Support for import/export of instance-level domain blocks/allows for 4.x w/ additional fixes (#20597)
* Allow import/export of instance-level domain blocks/allows (#1754) * Allow import/export of instance-level domain blocks/allows. Fixes #15095 * Pacify circleci * Address simple code review feedback * Add headers to exported CSV * Extract common import/export functionality to AdminExportControllerConcern * Add additional fields to instance-blocked domain export * Address review feedback * Split instance domain block/allow import/export into separate pages/controllers * Address code review feedback * Pacify DeepSource * Work around Paperclip::HasAttachmentFile for Rails 6 * Fix deprecated API warning in export tests * Remove after_commit workaround (cherry picked from commitmain94e98864e3
) * Add confirmation page when importing blocked domains (#1773) * Move glitch-soc-specific strings to glitch-soc-specific locale files * Add confirmation page when importing blocked domains (cherry picked from commitb91196f4b7
) * Fix authorization check in domain blocks controller (cherry picked from commit7527937758
) * Fix error strings for domain blocks and email-domain blocks Corrected issue with non-error message used for Mastodon:NotPermittedError in Domain Blocks Corrected issue Domain Blocks using the Email Domain Blocks message on ActionContoller::ParameterMissing Corrected issue with Email Domain Blocks using the not_permitted string from "custom emojii's" * Ran i18n-tasks normalize to address test failure * Removed unused admin.export_domain_blocks.not_permitted string Removing unused string as indicated by Check i18n * Fix tests (cherry picked from commit9094c2f52c
) * Fix domain block export not exporting blocks with only media rejection (cherry picked from commit26ff48ee48
) * Fix various issues with domain block import - stop using Paperclip for processing domain allow/block imports - stop leaving temporary files - better error handling - assume CSV files are UTF-8-encoded (cherry picked from commit cad824d8f501b95377e4f0a957e5a00d517a1902) Co-authored-by: Levi Bard <taktaktaktaktaktaktaktaktaktak@gmail.com> Co-authored-by: Claire <claire.github-309c@sitedethib.com>
parent
e7deea62d1
commit
c373148b3d
@ -0,0 +1,60 @@
|
|||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
require 'csv'
|
||||||
|
|
||||||
|
module Admin
|
||||||
|
class ExportDomainAllowsController < BaseController
|
||||||
|
include AdminExportControllerConcern
|
||||||
|
|
||||||
|
before_action :set_dummy_import!, only: [:new]
|
||||||
|
|
||||||
|
def new
|
||||||
|
authorize :domain_allow, :create?
|
||||||
|
end
|
||||||
|
|
||||||
|
def export
|
||||||
|
authorize :instance, :index?
|
||||||
|
send_export_file
|
||||||
|
end
|
||||||
|
|
||||||
|
def import
|
||||||
|
authorize :domain_allow, :create?
|
||||||
|
begin
|
||||||
|
@import = Admin::Import.new(import_params)
|
||||||
|
return render :new unless @import.validate
|
||||||
|
|
||||||
|
parse_import_data!(export_headers)
|
||||||
|
|
||||||
|
@data.take(Admin::Import::ROWS_PROCESSING_LIMIT).each do |row|
|
||||||
|
domain = row['#domain'].strip
|
||||||
|
next if DomainAllow.allowed?(domain)
|
||||||
|
|
||||||
|
domain_allow = DomainAllow.new(domain: domain)
|
||||||
|
log_action :create, domain_allow if domain_allow.save
|
||||||
|
end
|
||||||
|
flash[:notice] = I18n.t('admin.domain_allows.created_msg')
|
||||||
|
rescue ActionController::ParameterMissing
|
||||||
|
flash[:error] = I18n.t('admin.export_domain_allows.no_file')
|
||||||
|
end
|
||||||
|
redirect_to admin_instances_path
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def export_filename
|
||||||
|
'domain_allows.csv'
|
||||||
|
end
|
||||||
|
|
||||||
|
def export_headers
|
||||||
|
%w(#domain)
|
||||||
|
end
|
||||||
|
|
||||||
|
def export_data
|
||||||
|
CSV.generate(headers: export_headers, write_headers: true) do |content|
|
||||||
|
DomainAllow.allowed_domains.each do |instance|
|
||||||
|
content << [instance.domain]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,71 @@
|
|||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
require 'csv'
|
||||||
|
|
||||||
|
module Admin
|
||||||
|
class ExportDomainBlocksController < BaseController
|
||||||
|
include AdminExportControllerConcern
|
||||||
|
|
||||||
|
before_action :set_dummy_import!, only: [:new]
|
||||||
|
|
||||||
|
def new
|
||||||
|
authorize :domain_block, :create?
|
||||||
|
end
|
||||||
|
|
||||||
|
def export
|
||||||
|
authorize :instance, :index?
|
||||||
|
send_export_file
|
||||||
|
end
|
||||||
|
|
||||||
|
def import
|
||||||
|
authorize :domain_block, :create?
|
||||||
|
|
||||||
|
@import = Admin::Import.new(import_params)
|
||||||
|
return render :new unless @import.validate
|
||||||
|
|
||||||
|
parse_import_data!(export_headers)
|
||||||
|
|
||||||
|
@global_private_comment = I18n.t('admin.export_domain_blocks.import.private_comment_template', source: @import.data_file_name, date: I18n.l(Time.now.utc))
|
||||||
|
|
||||||
|
@form = Form::DomainBlockBatch.new
|
||||||
|
@domain_blocks = @data.take(Admin::Import::ROWS_PROCESSING_LIMIT).filter_map do |row|
|
||||||
|
domain = row['#domain'].strip
|
||||||
|
next if DomainBlock.rule_for(domain).present?
|
||||||
|
|
||||||
|
domain_block = DomainBlock.new(domain: domain,
|
||||||
|
severity: row['#severity'].strip,
|
||||||
|
reject_media: row['#reject_media'].strip,
|
||||||
|
reject_reports: row['#reject_reports'].strip,
|
||||||
|
private_comment: @global_private_comment,
|
||||||
|
public_comment: row['#public_comment']&.strip,
|
||||||
|
obfuscate: row['#obfuscate'].strip)
|
||||||
|
|
||||||
|
domain_block if domain_block.valid?
|
||||||
|
end
|
||||||
|
|
||||||
|
@warning_domains = Instance.where(domain: @domain_blocks.map(&:domain)).where('EXISTS (SELECT 1 FROM follows JOIN accounts ON follows.account_id = accounts.id OR follows.target_account_id = accounts.id WHERE accounts.domain = instances.domain)').pluck(:domain)
|
||||||
|
rescue ActionController::ParameterMissing
|
||||||
|
flash.now[:alert] = I18n.t('admin.export_domain_blocks.no_file')
|
||||||
|
set_dummy_import!
|
||||||
|
render :new
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def export_filename
|
||||||
|
'domain_blocks.csv'
|
||||||
|
end
|
||||||
|
|
||||||
|
def export_headers
|
||||||
|
%w(#domain #severity #reject_media #reject_reports #public_comment #obfuscate)
|
||||||
|
end
|
||||||
|
|
||||||
|
def export_data
|
||||||
|
CSV.generate(headers: export_headers, write_headers: true) do |content|
|
||||||
|
DomainBlock.with_limitations.each do |instance|
|
||||||
|
content << [instance.domain, instance.severity, instance.reject_media, instance.reject_reports, instance.public_comment, instance.obfuscate]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,39 @@
|
|||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
module AdminExportControllerConcern
|
||||||
|
extend ActiveSupport::Concern
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def send_export_file
|
||||||
|
respond_to do |format|
|
||||||
|
format.csv { send_data export_data, filename: export_filename }
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def export_data
|
||||||
|
raise 'Override in controller'
|
||||||
|
end
|
||||||
|
|
||||||
|
def export_filename
|
||||||
|
raise 'Override in controller'
|
||||||
|
end
|
||||||
|
|
||||||
|
def set_dummy_import!
|
||||||
|
@import = Admin::Import.new
|
||||||
|
end
|
||||||
|
|
||||||
|
def import_params
|
||||||
|
params.require(:admin_import).permit(:data)
|
||||||
|
end
|
||||||
|
|
||||||
|
def import_data_path
|
||||||
|
params[:admin_import][:data].path
|
||||||
|
end
|
||||||
|
|
||||||
|
def parse_import_data!(default_headers)
|
||||||
|
data = CSV.read(import_data_path, headers: true, encoding: 'UTF-8')
|
||||||
|
data = CSV.read(import_data_path, headers: default_headers, encoding: 'UTF-8') unless data.headers&.first&.strip&.include?(default_headers[0])
|
||||||
|
@data = data.reject(&:blank?)
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,32 @@
|
|||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
# A non-activerecord helper class for csv upload
|
||||||
|
class Admin::Import
|
||||||
|
include ActiveModel::Model
|
||||||
|
|
||||||
|
ROWS_PROCESSING_LIMIT = 20_000
|
||||||
|
|
||||||
|
attr_accessor :data
|
||||||
|
|
||||||
|
validates :data, presence: true
|
||||||
|
validate :validate_data
|
||||||
|
|
||||||
|
def data_file_name
|
||||||
|
data.original_filename
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def validate_data
|
||||||
|
return if data.blank?
|
||||||
|
|
||||||
|
csv_data = CSV.read(data.path, encoding: 'UTF-8')
|
||||||
|
|
||||||
|
row_count = csv_data.size
|
||||||
|
row_count -= 1 if csv_data.first&.first == '#domain'
|
||||||
|
|
||||||
|
errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if row_count > ROWS_PROCESSING_LIMIT
|
||||||
|
rescue CSV::MalformedCSVError => e
|
||||||
|
errors.add(:data, I18n.t('imports.errors.invalid_csv_file', error: e.message))
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,35 @@
|
|||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
class Form::DomainBlockBatch
|
||||||
|
include ActiveModel::Model
|
||||||
|
include Authorization
|
||||||
|
include AccountableConcern
|
||||||
|
|
||||||
|
attr_accessor :domain_blocks_attributes, :action, :current_account
|
||||||
|
|
||||||
|
def save
|
||||||
|
case action
|
||||||
|
when 'save'
|
||||||
|
save!
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def domain_blocks
|
||||||
|
@domain_blocks ||= domain_blocks_attributes.values.filter_map do |attributes|
|
||||||
|
DomainBlock.new(attributes.without('enabled')) if ActiveModel::Type::Boolean.new.cast(attributes['enabled'])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def save!
|
||||||
|
domain_blocks.each do |domain_block|
|
||||||
|
authorize(domain_block, :create?)
|
||||||
|
next if DomainBlock.rule_for(domain_block.domain).present?
|
||||||
|
|
||||||
|
domain_block.save!
|
||||||
|
DomainBlockWorker.perform_async(domain_block.id)
|
||||||
|
log_action :create, domain_block
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,10 @@
|
|||||||
|
- content_for :page_title do
|
||||||
|
= t('.title')
|
||||||
|
|
||||||
|
= simple_form_for @import, url: import_admin_export_domain_allows_path, html: { multipart: true } do |f|
|
||||||
|
.fields-row
|
||||||
|
.fields-group.fields-row__column.fields-row__column-6
|
||||||
|
= f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data'), as: :file
|
||||||
|
|
||||||
|
.actions
|
||||||
|
= f.button :button, t('imports.upload'), type: :submit
|
@ -0,0 +1,27 @@
|
|||||||
|
- existing_relationships ||= false
|
||||||
|
|
||||||
|
.batch-table__row{ class: [existing_relationships && 'batch-table__row--attention'] }
|
||||||
|
%label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
|
||||||
|
= f.check_box :enabled, checked: !existing_relationships
|
||||||
|
.batch-table__row__content.pending-account
|
||||||
|
.pending-account__header
|
||||||
|
%strong
|
||||||
|
= f.object.domain
|
||||||
|
= f.hidden_field :domain
|
||||||
|
= f.hidden_field :severity
|
||||||
|
= f.hidden_field :reject_media
|
||||||
|
= f.hidden_field :reject_reports
|
||||||
|
= f.hidden_field :obfuscate
|
||||||
|
= f.hidden_field :private_comment
|
||||||
|
= f.hidden_field :public_comment
|
||||||
|
|
||||||
|
%br/
|
||||||
|
|
||||||
|
= f.object.policies.map { |policy| t(policy, scope: 'admin.instances.content_policies.policies') }.join(' • ')
|
||||||
|
- if f.object.public_comment.present?
|
||||||
|
•
|
||||||
|
= f.object.public_comment
|
||||||
|
- if existing_relationships
|
||||||
|
•
|
||||||
|
= fa_icon 'warning fw'
|
||||||
|
= t('admin.export_domain_blocks.import.existing_relationships_warning')
|
@ -0,0 +1,21 @@
|
|||||||
|
- content_for :page_title do
|
||||||
|
= t('admin.export_domain_blocks.import.title')
|
||||||
|
|
||||||
|
%p= t('admin.export_domain_blocks.import.description_html')
|
||||||
|
|
||||||
|
- if defined?(@global_private_comment) && @global_private_comment.present?
|
||||||
|
%p= t('admin.export_domain_blocks.import.private_comment_description_html', comment: @global_private_comment)
|
||||||
|
|
||||||
|
= form_for(@form, url: batch_admin_domain_blocks_path) do |f|
|
||||||
|
.batch-table
|
||||||
|
.batch-table__toolbar
|
||||||
|
%label.batch-table__toolbar__select.batch-checkbox-all
|
||||||
|
= check_box_tag :batch_checkbox_all, nil, false
|
||||||
|
.batch-table__toolbar__actions
|
||||||
|
= f.button safe_join([fa_icon('copy'), t('admin.domain_blocks.import')]), name: :save, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
|
||||||
|
.batch-table__body
|
||||||
|
- if @domain_blocks.empty?
|
||||||
|
= nothing_here 'nothing-here--under-tabs'
|
||||||
|
- else
|
||||||
|
= f.simple_fields_for :domain_blocks, @domain_blocks do |ff|
|
||||||
|
= render 'domain_block', f: ff, existing_relationships: @warning_domains.include?(ff.object.domain)
|
@ -0,0 +1,10 @@
|
|||||||
|
- content_for :page_title do
|
||||||
|
= t('.title')
|
||||||
|
|
||||||
|
= simple_form_for @import, url: import_admin_export_domain_blocks_path, html: { multipart: true } do |f|
|
||||||
|
.fields-row
|
||||||
|
.fields-group.fields-row__column.fields-row__column-6
|
||||||
|
= f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data'), as: :file
|
||||||
|
|
||||||
|
.actions
|
||||||
|
= f.button :button, t('imports.upload'), type: :submit
|
@ -0,0 +1,48 @@
|
|||||||
|
require 'rails_helper'
|
||||||
|
|
||||||
|
RSpec.describe Admin::DomainAllowsController, type: :controller do
|
||||||
|
render_views
|
||||||
|
|
||||||
|
before do
|
||||||
|
sign_in Fabricate(:user, role: UserRole.find_by(name: 'Admin')), scope: :user
|
||||||
|
end
|
||||||
|
|
||||||
|
describe 'GET #new' do
|
||||||
|
it 'assigns a new domain allow' do
|
||||||
|
get :new
|
||||||
|
|
||||||
|
expect(assigns(:domain_allow)).to be_instance_of(DomainAllow)
|
||||||
|
expect(response).to have_http_status(200)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe 'POST #create' do
|
||||||
|
it 'blocks the domain when succeeded to save' do
|
||||||
|
post :create, params: { domain_allow: { domain: 'example.com' } }
|
||||||
|
|
||||||
|
expect(flash[:notice]).to eq I18n.t('admin.domain_allows.created_msg')
|
||||||
|
expect(response).to redirect_to(admin_instances_path)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'renders new when failed to save' do
|
||||||
|
Fabricate(:domain_allow, domain: 'example.com')
|
||||||
|
|
||||||
|
post :create, params: { domain_allow: { domain: 'example.com' } }
|
||||||
|
|
||||||
|
expect(response).to render_template :new
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe 'DELETE #destroy' do
|
||||||
|
it 'disallows the domain' do
|
||||||
|
service = double(call: true)
|
||||||
|
allow(UnallowDomainService).to receive(:new).and_return(service)
|
||||||
|
domain_allow = Fabricate(:domain_allow)
|
||||||
|
delete :destroy, params: { id: domain_allow.id }
|
||||||
|
|
||||||
|
expect(service).to have_received(:call).with(domain_allow)
|
||||||
|
expect(flash[:notice]).to eq I18n.t('admin.domain_allows.destroyed_msg')
|
||||||
|
expect(response).to redirect_to(admin_instances_path)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,42 @@
|
|||||||
|
require 'rails_helper'
|
||||||
|
|
||||||
|
RSpec.describe Admin::ExportDomainAllowsController, type: :controller do
|
||||||
|
render_views
|
||||||
|
|
||||||
|
before do
|
||||||
|
sign_in Fabricate(:user, role: UserRole.find_by(name: 'Admin')), scope: :user
|
||||||
|
end
|
||||||
|
|
||||||
|
describe 'GET #export' do
|
||||||
|
it 'renders instances' do
|
||||||
|
Fabricate(:domain_allow, domain: 'good.domain')
|
||||||
|
Fabricate(:domain_allow, domain: 'better.domain')
|
||||||
|
|
||||||
|
get :export, params: { format: :csv }
|
||||||
|
expect(response).to have_http_status(200)
|
||||||
|
expect(response.body).to eq(IO.read(File.join(file_fixture_path, 'domain_allows.csv')))
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe 'POST #import' do
|
||||||
|
it 'allows imported domains' do
|
||||||
|
post :import, params: { admin_import: { data: fixture_file_upload('domain_allows.csv') } }
|
||||||
|
|
||||||
|
expect(response).to redirect_to(admin_instances_path)
|
||||||
|
|
||||||
|
# Header should not be imported
|
||||||
|
expect(DomainAllow.where(domain: '#domain').present?).to eq(false)
|
||||||
|
|
||||||
|
# Domains should now be added
|
||||||
|
get :export, params: { format: :csv }
|
||||||
|
expect(response).to have_http_status(200)
|
||||||
|
expect(response.body).to eq(IO.read(File.join(file_fixture_path, 'domain_allows.csv')))
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'displays error on no file selected' do
|
||||||
|
post :import, params: { admin_import: {} }
|
||||||
|
expect(response).to redirect_to(admin_instances_path)
|
||||||
|
expect(flash[:error]).to eq(I18n.t('admin.export_domain_allows.no_file'))
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,35 @@
|
|||||||
|
require 'rails_helper'
|
||||||
|
|
||||||
|
RSpec.describe Admin::ExportDomainBlocksController, type: :controller do
|
||||||
|
render_views
|
||||||
|
|
||||||
|
before do
|
||||||
|
sign_in Fabricate(:user, role: UserRole.find_by(name: 'Admin')), scope: :user
|
||||||
|
end
|
||||||
|
|
||||||
|
describe 'GET #export' do
|
||||||
|
it 'renders instances' do
|
||||||
|
Fabricate(:domain_block, domain: 'bad.domain', severity: 'silence', public_comment: 'bad')
|
||||||
|
Fabricate(:domain_block, domain: 'worse.domain', severity: 'suspend', reject_media: true, reject_reports: true, public_comment: 'worse', obfuscate: true)
|
||||||
|
Fabricate(:domain_block, domain: 'reject.media', severity: 'noop', reject_media: true, public_comment: 'reject media')
|
||||||
|
Fabricate(:domain_block, domain: 'no.op', severity: 'noop', public_comment: 'noop')
|
||||||
|
|
||||||
|
get :export, params: { format: :csv }
|
||||||
|
expect(response).to have_http_status(200)
|
||||||
|
expect(response.body).to eq(IO.read(File.join(file_fixture_path, 'domain_blocks.csv')))
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe 'POST #import' do
|
||||||
|
it 'blocks imported domains' do
|
||||||
|
post :import, params: { admin_import: { data: fixture_file_upload('domain_blocks.csv') } }
|
||||||
|
|
||||||
|
expect(assigns(:domain_blocks).map(&:domain)).to match_array ['bad.domain', 'worse.domain', 'reject.media']
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'displays error on no file selected' do
|
||||||
|
post :import, params: { admin_import: {} }
|
||||||
|
expect(flash[:alert]).to eq(I18n.t('admin.export_domain_blocks.no_file'))
|
||||||
|
end
|
||||||
|
end
|
|
|
Loading…
Reference in new issue