1
0
Fork 0
mirror of https://github.com/mastodon/mastodon.git synced 2024-08-20 21:08:15 -07:00
This commit is contained in:
Matt Jankowski 2024-07-31 14:07:09 +00:00 committed by GitHub
commit 290fef4f09
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 13 additions and 576 deletions

View file

@ -45,7 +45,6 @@ class Admin::Metrics::Dimension::SpaceUsageDimension < Admin::Metrics::Dimension
PreviewCard.sum(:image_file_size),
Account.sum(Arel.sql('COALESCE(avatar_file_size, 0) + COALESCE(header_file_size, 0)')),
Backup.sum(:dump_file_size),
Import.sum(:data_file_size),
SiteUpload.sum(:file_file_size),
].sum

View file

@ -1,46 +0,0 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: imports
#
# id :bigint(8) not null, primary key
# type :integer not null
# approved :boolean default(FALSE), not null
# created_at :datetime not null
# updated_at :datetime not null
# data_file_name :string
# data_content_type :string
# data_file_size :integer
# data_updated_at :datetime
# account_id :bigint(8) not null
# overwrite :boolean default(FALSE), not null
#
# NOTE: This is a deprecated model, only kept to not break ongoing imports
# on upgrade. See `BulkImport` and `Form::Import` for its replacements.
class Import < ApplicationRecord
FILE_TYPES = %w(text/plain text/csv application/csv).freeze
MODES = %i(merge overwrite).freeze
self.inheritance_column = false
belongs_to :account
enum :type, { following: 0, blocking: 1, muting: 2, domain_blocking: 3, bookmarks: 4 }
validates :type, presence: true
has_attached_file :data
validates_attachment_content_type :data, content_type: FILE_TYPES
validates_attachment_presence :data
def mode
overwrite? ? :overwrite : :merge
end
def mode=(str)
self.overwrite = str.to_sym == :overwrite
end
end

View file

@ -1,144 +0,0 @@
# frozen_string_literal: true
require 'csv'
# NOTE: This is a deprecated service, only kept to not break ongoing imports
# on upgrade. See `BulkImportService` for its replacement.
class ImportService < BaseService
ROWS_PROCESSING_LIMIT = 20_000
def call(import)
@import = import
@account = @import.account
case @import.type
when 'following'
import_follows!
when 'blocking'
import_blocks!
when 'muting'
import_mutes!
when 'domain_blocking'
import_domain_blocks!
when 'bookmarks'
import_bookmarks!
end
end
private
def import_follows!
parse_import_data!(['Account address'])
import_relationships!('follow', 'unfollow', @account.following, ROWS_PROCESSING_LIMIT, reblogs: { header: 'Show boosts', default: true }, notify: { header: 'Notify on new posts', default: false }, languages: { header: 'Languages', default: nil })
end
def import_blocks!
parse_import_data!(['Account address'])
import_relationships!('block', 'unblock', @account.blocking, ROWS_PROCESSING_LIMIT)
end
def import_mutes!
parse_import_data!(['Account address'])
import_relationships!('mute', 'unmute', @account.muting, ROWS_PROCESSING_LIMIT, notifications: { header: 'Hide notifications', default: true })
end
def import_domain_blocks!
parse_import_data!(['#domain'])
items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row['#domain'].strip }
if @import.overwrite?
presence_hash = items.index_with(true)
@account.domain_blocks.find_each do |domain_block|
if presence_hash[domain_block.domain]
items.delete(domain_block.domain)
else
@account.unblock_domain!(domain_block.domain)
end
end
end
items.each do |domain|
@account.block_domain!(domain)
end
AfterAccountDomainBlockWorker.push_bulk(items) do |domain|
[@account.id, domain]
end
end
def import_relationships!(action, undo_action, overwrite_scope, limit, extra_fields = {})
local_domain_suffix = "@#{Rails.configuration.x.local_domain}"
items = @data.take(limit).map { |row| [row['Account address']&.strip&.delete_suffix(local_domain_suffix), extra_fields.to_h { |key, field_settings| [key, row[field_settings[:header]]&.strip || field_settings[:default]] }] }.reject { |(id, _)| id.blank? }
if @import.overwrite?
presence_hash = items.each_with_object({}) { |(id, extra), mapping| mapping[id] = [true, extra] }
overwrite_scope.reorder(nil).find_each do |target_account|
if presence_hash[target_account.acct]
items.delete(target_account.acct)
extra = presence_hash[target_account.acct][1]
Import::RelationshipWorker.perform_async(@account.id, target_account.acct, action, extra.stringify_keys)
else
Import::RelationshipWorker.perform_async(@account.id, target_account.acct, undo_action)
end
end
end
head_items = items.uniq { |acct, _| acct.split('@')[1] }
tail_items = items - head_items
Import::RelationshipWorker.push_bulk(head_items + tail_items) do |acct, extra|
[@account.id, acct, action, extra.stringify_keys]
end
end
def import_bookmarks!
parse_import_data!(['#uri'])
items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row['#uri'].strip }
if @import.overwrite?
presence_hash = items.index_with(true)
@account.bookmarks.find_each do |bookmark|
if presence_hash[bookmark.status.uri]
items.delete(bookmark.status.uri)
else
bookmark.destroy!
end
end
end
statuses = items.filter_map do |uri|
status = ActivityPub::TagManager.instance.uri_to_resource(uri, Status)
next if status.nil? && ActivityPub::TagManager.instance.local_uri?(uri)
status || ActivityPub::FetchRemoteStatusService.new.call(uri)
rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::UnexpectedResponseError
nil
rescue => e
Rails.logger.warn "Unexpected error when importing bookmark: #{e}"
nil
end
account_ids = statuses.map(&:account_id)
preloaded_relations = @account.relations_map(account_ids, skip_blocking_and_muting: true)
statuses.keep_if { |status| StatusPolicy.new(@account, status, preloaded_relations).show? }
statuses.each do |status|
@account.bookmarks.find_or_create_by!(account: @account, status: status)
end
end
def parse_import_data!(default_headers)
data = CSV.parse(import_data, headers: true)
data = CSV.parse(import_data, headers: default_headers) unless data.headers&.first&.strip&.include?(' ')
@data = data.compact_blank
end
def import_data
Paperclip.io_adapters.for(@import.data).read.force_encoding(Encoding::UTF_8)
end
end

View file

@ -23,7 +23,7 @@
= f.input :mode,
as: :radio_buttons,
collection_wrapper_tag: 'ul',
collection: Import::MODES,
collection: Form::Import::MODES,
item_wrapper_tag: 'li',
label_method: ->(mode) { safe_join([I18n.t("imports.modes.#{mode}"), content_tag(:span, I18n.t("imports.modes.#{mode}_long"), class: 'hint')]) }

View file

@ -1,57 +0,0 @@
# frozen_string_literal: true
# NOTE: This is a deprecated worker, only kept to not break ongoing imports
# on upgrade. See `Import::RowWorker` for its replacement.
class Import::RelationshipWorker
include Sidekiq::Worker
sidekiq_options queue: 'pull', retry: 8, dead: false
def perform(account_id, target_account_uri, relationship, options)
from_account = Account.find(account_id)
target_domain = domain(target_account_uri)
target_account = stoplight_wrapper(target_domain).run { ResolveAccountService.new.call(target_account_uri, { check_delivery_availability: true }) }
options.symbolize_keys!
return if target_account.nil?
case relationship
when 'follow'
begin
FollowService.new.call(from_account, target_account, **options)
rescue ActiveRecord::RecordInvalid
raise if FollowLimitValidator.limit_for_account(from_account) < from_account.following_count
end
when 'unfollow'
UnfollowService.new.call(from_account, target_account)
when 'block'
BlockService.new.call(from_account, target_account)
when 'unblock'
UnblockService.new.call(from_account, target_account)
when 'mute'
MuteService.new.call(from_account, target_account, **options)
when 'unmute'
UnmuteService.new.call(from_account, target_account)
end
rescue ActiveRecord::RecordNotFound
true
end
def domain(uri)
domain = uri.is_a?(Account) ? uri.domain : uri.split('@')[1]
TagManager.instance.local_domain?(domain) ? nil : TagManager.instance.normalize_domain(domain)
end
def stoplight_wrapper(domain)
if domain.present?
Stoplight("source:#{domain}")
.with_fallback { nil }
.with_threshold(1)
.with_cool_off_time(5.minutes.seconds)
.with_error_handler { |error, handle| error.is_a?(HTTP::Error) || error.is_a?(OpenSSL::SSL::SSLError) ? handle.call(error) : raise(error) }
else
Stoplight('domain-blank')
end
end
end

View file

@ -1,17 +0,0 @@
# frozen_string_literal: true
# NOTE: This is a deprecated worker, only kept to not break ongoing imports
# on upgrade. See `ImportWorker` for its replacement.
class ImportWorker
include Sidekiq::Worker
sidekiq_options queue: 'pull', retry: false
def perform(import_id)
import = Import.find(import_id)
ImportService.new.call(import)
ensure
import&.destroy
end
end

View file

@ -168,7 +168,7 @@ else
end
Rails.application.reloader.to_prepare do
Paperclip.options[:content_type_mappings] = { csv: Import::FILE_TYPES }
Paperclip.options[:content_type_mappings] = { csv: %w(text/plain text/csv application/csv) }
end
# In some places in the code, we rescue this exception, but we don't always

View file

@ -0,0 +1,11 @@
# frozen_string_literal: true
class DropImports < ActiveRecord::Migration[7.1]
def up
drop_table :imports
end
def down
raise ActiveRecord::IrreversibleMigration
end
end

View file

@ -551,19 +551,6 @@ ActiveRecord::Schema[7.1].define(version: 2024_07_24_181224) do
t.index ["user_id"], name: "index_identities_on_user_id"
end
create_table "imports", force: :cascade do |t|
t.integer "type", null: false
t.boolean "approved", default: false, null: false
t.datetime "created_at", precision: nil, null: false
t.datetime "updated_at", precision: nil, null: false
t.string "data_file_name"
t.string "data_content_type"
t.integer "data_file_size"
t.datetime "data_updated_at", precision: nil
t.bigint "account_id", null: false
t.boolean "overwrite", default: false, null: false
end
create_table "invites", force: :cascade do |t|
t.bigint "user_id", null: false
t.string "code", default: "", null: false
@ -1322,7 +1309,6 @@ ActiveRecord::Schema[7.1].define(version: 2024_07_24_181224) do
add_foreign_key "follows", "accounts", name: "fk_32ed1b5560", on_delete: :cascade
add_foreign_key "generated_annual_reports", "accounts"
add_foreign_key "identities", "users", name: "fk_bea040f377", on_delete: :cascade
add_foreign_key "imports", "accounts", name: "fk_6db1b6e408", on_delete: :cascade
add_foreign_key "invites", "users", on_delete: :cascade
add_foreign_key "list_accounts", "accounts", on_delete: :cascade
add_foreign_key "list_accounts", "follow_requests", on_delete: :cascade

View file

@ -284,7 +284,6 @@ module Mastodon::CLI
say("Avatars:\t#{number_to_human_size(Account.sum(:avatar_file_size))} (#{number_to_human_size(Account.local.sum(:avatar_file_size))} local)")
say("Headers:\t#{number_to_human_size(Account.sum(:header_file_size))} (#{number_to_human_size(Account.local.sum(:header_file_size))} local)")
say("Backups:\t#{number_to_human_size(Backup.sum(:dump_file_size))}")
say("Imports:\t#{number_to_human_size(Import.sum(:data_file_size))}")
say("Settings:\t#{number_to_human_size(SiteUpload.sum(:file_file_size))}")
end
@ -338,7 +337,6 @@ module Mastodon::CLI
Account
Backup
CustomEmoji
Import
MediaAttachment
PreviewCard
SiteUpload

View file

@ -1,7 +0,0 @@
# frozen_string_literal: true
Fabricator(:import) do
account
type :following
data { attachment_fixture('imports.txt') }
end

View file

@ -1,21 +0,0 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe Import do
let(:account) { Fabricate(:account) }
let(:type) { 'following' }
let(:data) { attachment_fixture('imports.txt') }
describe 'validations' do
it 'is invalid without an type' do
import = described_class.create(account: account, data: data)
expect(import).to model_have_error_on_field(:type)
end
it 'is invalid without a data' do
import = described_class.create(account: account, type: type)
expect(import).to model_have_error_on_field(:data)
end
end
end

View file

@ -1,242 +0,0 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe ImportService, :inline_jobs do
include RoutingHelper
let!(:account) { Fabricate(:account, locked: false) }
let!(:bob) { Fabricate(:account, username: 'bob', locked: false) }
let!(:eve) { Fabricate(:account, username: 'eve', domain: 'example.com', locked: false, protocol: :activitypub, inbox_url: 'https://example.com/inbox') }
before do
stub_request(:post, 'https://example.com/inbox').to_return(status: 200)
end
context 'when importing old-style list of muted users' do
subject { described_class.new }
let(:csv) { attachment_fixture('mute-imports.txt') }
describe 'when no accounts are muted' do
let(:import) { Import.create(account: account, type: 'muting', data: csv) }
it 'mutes the listed accounts, including notifications' do
subject.call(import)
expect(account.muting.count).to eq 2
expect(Mute.find_by(account: account, target_account: bob).hide_notifications).to be true
end
end
describe 'when some accounts are muted and overwrite is not set' do
let(:import) { Import.create(account: account, type: 'muting', data: csv) }
it 'mutes the listed accounts, including notifications' do
account.mute!(bob, notifications: false)
subject.call(import)
expect(account.muting.count).to eq 2
expect(Mute.find_by(account: account, target_account: bob).hide_notifications).to be true
end
end
describe 'when some accounts are muted and overwrite is set' do
let(:import) { Import.create(account: account, type: 'muting', data: csv, overwrite: true) }
it 'mutes the listed accounts, including notifications' do
account.mute!(bob, notifications: false)
subject.call(import)
expect(account.muting.count).to eq 2
expect(Mute.find_by(account: account, target_account: bob).hide_notifications).to be true
end
end
end
context 'when importing new-style list of muted users' do
subject { described_class.new }
let(:csv) { attachment_fixture('new-mute-imports.txt') }
describe 'when no accounts are muted' do
let(:import) { Import.create(account: account, type: 'muting', data: csv) }
it 'mutes the listed accounts, respecting notifications' do
subject.call(import)
expect(account.muting.count).to eq 2
expect(Mute.find_by(account: account, target_account: bob).hide_notifications).to be true
expect(Mute.find_by(account: account, target_account: eve).hide_notifications).to be false
end
end
describe 'when some accounts are muted and overwrite is not set' do
let(:import) { Import.create(account: account, type: 'muting', data: csv) }
it 'mutes the listed accounts, respecting notifications' do
account.mute!(bob, notifications: true)
subject.call(import)
expect(account.muting.count).to eq 2
expect(Mute.find_by(account: account, target_account: bob).hide_notifications).to be true
expect(Mute.find_by(account: account, target_account: eve).hide_notifications).to be false
end
end
describe 'when some accounts are muted and overwrite is set' do
let(:import) { Import.create(account: account, type: 'muting', data: csv, overwrite: true) }
it 'mutes the listed accounts, respecting notifications' do
account.mute!(bob, notifications: true)
subject.call(import)
expect(account.muting.count).to eq 2
expect(Mute.find_by(account: account, target_account: bob).hide_notifications).to be true
expect(Mute.find_by(account: account, target_account: eve).hide_notifications).to be false
end
end
end
context 'when importing old-style list of followed users' do
subject { described_class.new }
let(:csv) { attachment_fixture('mute-imports.txt') }
describe 'when no accounts are followed' do
let(:import) { Import.create(account: account, type: 'following', data: csv) }
it 'follows the listed accounts, including boosts' do
subject.call(import)
expect(account.following.count).to eq 1
expect(account.follow_requests.count).to eq 1
expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
end
end
describe 'when some accounts are already followed and overwrite is not set' do
let(:import) { Import.create(account: account, type: 'following', data: csv) }
it 'follows the listed accounts, including notifications' do
account.follow!(bob, reblogs: false)
subject.call(import)
expect(account.following.count).to eq 1
expect(account.follow_requests.count).to eq 1
expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
end
end
describe 'when some accounts are already followed and overwrite is set' do
let(:import) { Import.create(account: account, type: 'following', data: csv, overwrite: true) }
it 'mutes the listed accounts, including notifications' do
account.follow!(bob, reblogs: false)
subject.call(import)
expect(account.following.count).to eq 1
expect(account.follow_requests.count).to eq 1
expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
end
end
end
context 'when importing new-style list of followed users' do
subject { described_class.new }
let(:csv) { attachment_fixture('new-following-imports.txt') }
describe 'when no accounts are followed' do
let(:import) { Import.create(account: account, type: 'following', data: csv) }
it 'follows the listed accounts, respecting boosts' do
subject.call(import)
expect(account.following.count).to eq 1
expect(account.follow_requests.count).to eq 1
expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
expect(FollowRequest.find_by(account: account, target_account: eve).show_reblogs).to be false
end
end
describe 'when some accounts are already followed and overwrite is not set' do
let(:import) { Import.create(account: account, type: 'following', data: csv) }
it 'mutes the listed accounts, respecting notifications' do
account.follow!(bob, reblogs: true)
subject.call(import)
expect(account.following.count).to eq 1
expect(account.follow_requests.count).to eq 1
expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
expect(FollowRequest.find_by(account: account, target_account: eve).show_reblogs).to be false
end
end
describe 'when some accounts are already followed and overwrite is set' do
let(:import) { Import.create(account: account, type: 'following', data: csv, overwrite: true) }
it 'mutes the listed accounts, respecting notifications' do
account.follow!(bob, reblogs: true)
subject.call(import)
expect(account.following.count).to eq 1
expect(account.follow_requests.count).to eq 1
expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
expect(FollowRequest.find_by(account: account, target_account: eve).show_reblogs).to be false
end
end
end
# Based on the bug report 20571 where UTF-8 encoded domains were rejecting import of their users
#
# https://github.com/mastodon/mastodon/issues/20571
context 'with a utf-8 encoded domains' do
subject { described_class.new }
let!(:nare) { Fabricate(:account, username: 'nare', domain: 'թութ.հայ', locked: false, protocol: :activitypub, inbox_url: 'https://թութ.հայ/inbox') }
let(:csv) { attachment_fixture('utf8-followers.txt') }
let(:import) { Import.create(account: account, type: 'following', data: csv) }
# Make sure to not actually go to the remote server
before do
stub_request(:post, nare.inbox_url).to_return(status: 200)
end
it 'follows the listed account' do
expect(account.follow_requests.count).to eq 0
subject.call(import)
expect(account.follow_requests.count).to eq 1
end
end
context 'when importing bookmarks' do
subject { described_class.new }
let(:csv) { attachment_fixture('bookmark-imports.txt') }
let(:local_account) { Fabricate(:account, username: 'foo', domain: '') }
let!(:remote_status) { Fabricate(:status, uri: 'https://example.com/statuses/1312') }
let!(:direct_status) { Fabricate(:status, uri: 'https://example.com/statuses/direct', visibility: :direct) }
around do |example|
local_before = Rails.configuration.x.local_domain
web_before = Rails.configuration.x.web_domain
Rails.configuration.x.local_domain = 'local.com'
Rails.configuration.x.web_domain = 'local.com'
example.run
Rails.configuration.x.web_domain = web_before
Rails.configuration.x.local_domain = local_before
end
before do
service = instance_double(ActivityPub::FetchRemoteStatusService)
allow(ActivityPub::FetchRemoteStatusService).to receive(:new).and_return(service)
allow(service).to receive(:call).with('https://unknown-remote.com/users/bar/statuses/1') do
Fabricate(:status, uri: 'https://unknown-remote.com/users/bar/statuses/1')
end
end
describe 'when no bookmarks are set' do
let(:import) { Import.create(account: account, type: 'bookmarks', data: csv) }
it 'adds the toots the user has access to to bookmarks' do
local_status = Fabricate(:status, account: local_account, uri: 'https://local.com/users/foo/statuses/42', id: 42, local: true)
subject.call(import)
expect(account.bookmarks.map { |bookmark| bookmark.status.id }).to include(local_status.id)
expect(account.bookmarks.map { |bookmark| bookmark.status.id }).to include(remote_status.id)
expect(account.bookmarks.map { |bookmark| bookmark.status.id }).to_not include(direct_status.id)
expect(account.bookmarks.count).to eq 3
end
end
end
end

View file

@ -1,23 +0,0 @@
# frozen_string_literal: true
require 'rails_helper'
describe ImportWorker do
let(:worker) { described_class.new }
let(:service) { instance_double(ImportService, call: true) }
describe '#perform' do
before do
allow(ImportService).to receive(:new).and_return(service)
end
let(:import) { Fabricate(:import) }
it 'sends the import to the service' do
worker.perform(import.id)
expect(service).to have_received(:call).with(import)
expect { import.reload }.to raise_error(ActiveRecord::RecordNotFound)
end
end
end