th: maybe deal with spam

th-downstream^2
Kouhai 2 years ago
parent bb4c06aa95
commit ecc917008e

@ -89,6 +89,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
resolve_thread(@status)
fetch_replies(@status)
return if Treehouse::Automod.process_status!(@status)
distribute
forward_for_reply
end

@ -1,5 +1,7 @@
# frozen_string_literal: true
require 'digest'
class ActivityPub::ProcessAccountService < BaseService
include JsonLdHelper
include DomainControlHelper
@ -90,6 +92,9 @@ class ActivityPub::ProcessAccountService < BaseService
set_immediate_protocol_attributes!
set_fetchable_key! unless @account.suspended? && @account.suspension_origin_local?
set_immediate_attributes! unless @account.suspended?
Treehouse::Automod.process_account!(@account)
set_fetchable_attributes! unless @options[:only_key] || @account.suspended?
@account.save_with_optional_media!

@ -40,6 +40,7 @@ class ReportService < BaseService
end
def notify_staff!
return if @options[:th_skip_notify_staff]
return if @report.unresolved_siblings?
User.those_who_can(:manage_reports).includes(:account).find_each do |u|
@ -65,6 +66,7 @@ class ReportService < BaseService
end
def forward?
return false if @options[:th_skip_forward]
!@target_account.local? && ActiveModel::Type::Boolean.new.cast(@options[:forward])
end

@ -52,6 +52,8 @@ require_relative '../lib/active_record/batches'
require_relative '../lib/simple_navigation/item_extensions'
require_relative '../lib/http_extensions'
require_relative '../lib/treehouse/automod'
Dotenv::Railtie.load
Bundler.require(:pam_authentication) if ENV['PAM_ENABLED'] == 'true'
@ -107,5 +109,9 @@ module Mastodon
Devise::FailureApp.include AbstractController::Callbacks
Devise::FailureApp.include Localized
end
config.x.th_automod.automod_account_username = ENV['TH_STAFF_ACCOUNT']
config.x.th_automod.account_service_heuristic_auto_suspend_active = ENV.fetch('TH_ACCOUNT_SERVICE_HEURISTIC_AUTO_SUSPEND', '') == 'that-one-spammer'
config.x.th_automod.mention_spam_heuristic_auto_limit_active = ENV.fetch('TH_MENTION_SPAM_HEURISTIC_AUTO_LIMIT_ACTIVE', '') == 'can-spam'
end
end

@ -0,0 +1,128 @@
module Treehouse
module Automod
COMMENT_HEADER = <<~EOS
Tracking Report - automatically created by TreehouseAutomod
EOS
WARNING_TEXT = <<~EOS
Tracking Infraction - automatically created by TreehouseAutomod
EOS
def self.suspend_with_tracking_report!(account, status_ids: [], explanation: "")
account.save!
self.file_tracking_report!(account, status_ids: status_ids) unless account.suspension_origin == "local"
account.suspend! unless account.suspension_origin == "local"
end
def self.file_tracking_report!(account, status_ids: [], explanation: "")
reporter = self.staff_account
return if reporter.nil?
report = ReportService.new.call(
reporter,
account,
{
status_ids: status_ids,
comment: explanation.blank? ? COMMENT_HEADER : "#{COMMENT_HEADER}\n\n#{EXPLANATION}",
th_skip_notify_staff: true,
th_skip_forward: true,
}
)
report.spam!
report.assign_to_self!(reporter)
account_action = Admin::AccountAction.new(
type: "suspend",
report_id: report.id,
target_account: account,
current_account: reporter,
send_email_notification: false,
text: WARNING_TEXT,
)
account_action.save!
report.resolve!(reporter)
end
def self.staff_account
username = Rails.configuration.x.th_automod.automod_account_username
Account.find_local(username) unless username.blank?
end
def self.process_status!(status)
ActivityPubActivityCreateExt.process!(status)
end
def self.process_account!(account)
AccountServiceExt.process!(account)
end
module ActivityPubActivityCreateExt
EXPLANATION = <<~EOS
This account was automatically suspended by TreehouseAutomod, an unsupported feature.
Currently, the account-only heuristic should only automatically suspend accounts with one specific username and display name.
If this action is unexpected, please unset TH_MENTION_SPAM_HEURISTIC_AUTO_LIMIT_ACTIVE.
EOS
# check if the status should be considered spam
# @return true if the status was reported and the account was infracted
def process!(status)
return false unless Rails.configuration.x.th_automod.mention_spam_heuristic_auto_limit_active
account = status.account
minimal_effort = account.note.blank? && account.avatar_remote_url.blank? && account.header_remote_url.blank?
return false if (account.local? ||
account.local_followers_account > 0 ||
!minimal_effort)
# minimal effort account, check mentions and account-known age
status.mentions.size > 8 && account.created_at > (Time.now - 1.day)
end
end
module AccountServiceExt
# hardcoded for now
# md5 because they don't deserve more mentions
HEURISTIC_NAMES = {
"0116a9deace3289b7092e945ef5ca0a5" => Set["57d3d0b932cc9cd01be6b2f4e82c1a4a"],
}
# probably mathematically impossible to collide, but just in case...
HEURISTIC_MAX_LEN = 16
EXPLANATION = <<~EOS
This account was automatically suspended by TreehouseAutomod, an unsupported feature.
Currently, the account-only heuristic should only automatically suspend accounts with one specific username and display name.
If this action is unexpected, please unset TH_HEURISTIC_AUTO_SUSPEND.
EOS
# @return true if the account was infracted
def self.process!(account)
return false unless heuristic_auto_suspend?(account)
Automod.suspend_with_tracking_report!(account, explanation: EXPLANATION) unless account.suspension_origin == "local"
true
end
def self.matches_evil_hash?(account)
username_md5 = Digest::MD5.hexdigest(account.username)
display_name_md5 = Digest::MD5.hexdigest(account.display_name)
HEURISTIC_NAMES[username_md5].include?(display_name_md5)
end
def self.heuristic_auto_suspend?(account)
return false unless Rails.configuration.x.th_automod.account_service_heuristic_auto_suspend_active
return unless account.username.length < HEURISTIC_MAX_LEN && account.display_name.length < HEURISTIC_MAX_LEN
self.matches_evil_hash?(account)
end
end
end
end

@ -200,6 +200,114 @@ RSpec.describe ActivityPub::ProcessAccountService, type: :service do
end
end
context 'treehouse automod' do
subject { described_class.new.call(account_username, 'foo.test', payload) }
let(:account_username) { 'evil' }
let(:account_display_name) { 'evil display name' }
let(:account_payload_suspended) { false }
let(:automod_account_username) { nil }
let(:payload) do
{
id: 'https://foo.test',
type: 'Actor',
inbox: 'https://foo.test/inbox',
suspended: account_payload_suspended,
name: account_display_name,
}.with_indifferent_access
end
let(:name_hash_hash) do
{
# 'evil' => 'evil display name'
'4034a346ccee15292d823416f7510a2f' => Set['225e44a7c4a792ee22a4ada2032da7cd']
}
end
before do
allow(Rails.configuration.x.th_automod).to receive(:account_service_heuristic_auto_suspend_active).and_return(true)
allow(Rails.configuration.x.th_automod).to receive(:automod_account_username).and_return(automod_account_username)
stub_const('::Treehouse::Automod::AccountServiceExt::HEURISTIC_NAMES', name_hash_hash)
stub_const('::Treehouse::Automod::AccountServiceExt::HEURISTIC_MAX_LEN', 20)
end
context 'new account' do
context 'heuristic matching' do
it 'suspends the user locally' do
expect(subject.suspended?).to be true
expect(subject.suspension_origin_local?).to be true
end
end
context 'heuristic not matching' do
let(:account_display_name) { '' }
it 'does nothing' do
expect(subject.suspended?).to be false
end
end
end
context 'existing account' do
let!(:account) { Fabricate(:account, username: account_username, domain: 'foo.test', display_name: account_display_name) }
before do
allow(Admin::SuspensionWorker).to receive(:perform_async)
end
context 'heuristic matching' do
it 'suspends the user locally' do
expect(subject.suspended?).to be true
expect(subject.suspension_origin_local?).to be true
end
end
context 'heuristic not matching' do
let(:account_display_name) { 'not evil display name' }
it 'does nothing' do
expect(subject.suspended?).to be false
end
context 'suspended locally' do
before do
account.suspend!(origin: :local)
end
it 'does nothing' do
expect(subject.suspended?).to be true
end
end
end
end
context 'tracking report' do
let(:automod_account_username) { 'automod_test' }
let!(:automod_user_role) { Fabricate(:user_role, name: 'Automod', permissions: UserRole::FLAGS[:administrator]) }
let!(:automod_account) do
account = Fabricate(:account, username: automod_account_username)
account.user.role_id = automod_user_role.id
account.user.save!
account
end
it 'creates report' do
expect(subject.targeted_reports.empty?).to be_falsy
report = Report.find_by(target_account_id: subject.id, account_id: automod_account.id, assigned_account_id: automod_account.id)
expect(report.comment.starts_with?('Tracking Report - automatically created by TreehouseAutomod')).to be_truthy
end
it 'creates account action' do
subject
expect(Admin::ActionLog.find_by(account_id: automod_account.id, target_id: subject.id)).not_to be nil
end
end
end
private
def create_some_remote_accounts
@ -209,4 +317,5 @@ RSpec.describe ActivityPub::ProcessAccountService, type: :service do
def create_fewer_than_rate_limit_accounts
change(Account.remote, :count).by_at_most(5)
end
end

Loading…
Cancel
Save