diff --git a/app/lib/admin/metrics/dimension/space_usage_dimension.rb b/app/lib/admin/metrics/dimension/space_usage_dimension.rb index f1b6dba0403..0d3fd8db33c 100644 --- a/app/lib/admin/metrics/dimension/space_usage_dimension.rb +++ b/app/lib/admin/metrics/dimension/space_usage_dimension.rb @@ -45,7 +45,6 @@ class Admin::Metrics::Dimension::SpaceUsageDimension < Admin::Metrics::Dimension PreviewCard.sum(:image_file_size), Account.sum(Arel.sql('COALESCE(avatar_file_size, 0) + COALESCE(header_file_size, 0)')), Backup.sum(:dump_file_size), - Import.sum(:data_file_size), SiteUpload.sum(:file_file_size), ].sum diff --git a/app/models/import.rb b/app/models/import.rb deleted file mode 100644 index 4bdb392014b..00000000000 --- a/app/models/import.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -# == Schema Information -# -# Table name: imports -# -# id :bigint(8) not null, primary key -# type :integer not null -# approved :boolean default(FALSE), not null -# created_at :datetime not null -# updated_at :datetime not null -# data_file_name :string -# data_content_type :string -# data_file_size :integer -# data_updated_at :datetime -# account_id :bigint(8) not null -# overwrite :boolean default(FALSE), not null -# - -# NOTE: This is a deprecated model, only kept to not break ongoing imports -# on upgrade. See `BulkImport` and `Form::Import` for its replacements. - -class Import < ApplicationRecord - FILE_TYPES = %w(text/plain text/csv application/csv).freeze - MODES = %i(merge overwrite).freeze - - self.inheritance_column = false - - belongs_to :account - - enum :type, { following: 0, blocking: 1, muting: 2, domain_blocking: 3, bookmarks: 4 } - - validates :type, presence: true - - has_attached_file :data - validates_attachment_content_type :data, content_type: FILE_TYPES - validates_attachment_presence :data - - def mode - overwrite? ? :overwrite : :merge - end - - def mode=(str) - self.overwrite = str.to_sym == :overwrite - end -end diff --git a/app/services/import_service.rb b/app/services/import_service.rb deleted file mode 100644 index 6dafb5a0bb1..00000000000 --- a/app/services/import_service.rb +++ /dev/null @@ -1,144 +0,0 @@ -# frozen_string_literal: true - -require 'csv' - -# NOTE: This is a deprecated service, only kept to not break ongoing imports -# on upgrade. See `BulkImportService` for its replacement. - -class ImportService < BaseService - ROWS_PROCESSING_LIMIT = 20_000 - - def call(import) - @import = import - @account = @import.account - - case @import.type - when 'following' - import_follows! - when 'blocking' - import_blocks! - when 'muting' - import_mutes! - when 'domain_blocking' - import_domain_blocks! - when 'bookmarks' - import_bookmarks! - end - end - - private - - def import_follows! - parse_import_data!(['Account address']) - import_relationships!('follow', 'unfollow', @account.following, ROWS_PROCESSING_LIMIT, reblogs: { header: 'Show boosts', default: true }, notify: { header: 'Notify on new posts', default: false }, languages: { header: 'Languages', default: nil }) - end - - def import_blocks! - parse_import_data!(['Account address']) - import_relationships!('block', 'unblock', @account.blocking, ROWS_PROCESSING_LIMIT) - end - - def import_mutes! - parse_import_data!(['Account address']) - import_relationships!('mute', 'unmute', @account.muting, ROWS_PROCESSING_LIMIT, notifications: { header: 'Hide notifications', default: true }) - end - - def import_domain_blocks! - parse_import_data!(['#domain']) - items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row['#domain'].strip } - - if @import.overwrite? - presence_hash = items.index_with(true) - - @account.domain_blocks.find_each do |domain_block| - if presence_hash[domain_block.domain] - items.delete(domain_block.domain) - else - @account.unblock_domain!(domain_block.domain) - end - end - end - - items.each do |domain| - @account.block_domain!(domain) - end - - AfterAccountDomainBlockWorker.push_bulk(items) do |domain| - [@account.id, domain] - end - end - - def import_relationships!(action, undo_action, overwrite_scope, limit, extra_fields = {}) - local_domain_suffix = "@#{Rails.configuration.x.local_domain}" - items = @data.take(limit).map { |row| [row['Account address']&.strip&.delete_suffix(local_domain_suffix), extra_fields.to_h { |key, field_settings| [key, row[field_settings[:header]]&.strip || field_settings[:default]] }] }.reject { |(id, _)| id.blank? } - - if @import.overwrite? - presence_hash = items.each_with_object({}) { |(id, extra), mapping| mapping[id] = [true, extra] } - - overwrite_scope.reorder(nil).find_each do |target_account| - if presence_hash[target_account.acct] - items.delete(target_account.acct) - extra = presence_hash[target_account.acct][1] - Import::RelationshipWorker.perform_async(@account.id, target_account.acct, action, extra.stringify_keys) - else - Import::RelationshipWorker.perform_async(@account.id, target_account.acct, undo_action) - end - end - end - - head_items = items.uniq { |acct, _| acct.split('@')[1] } - tail_items = items - head_items - - Import::RelationshipWorker.push_bulk(head_items + tail_items) do |acct, extra| - [@account.id, acct, action, extra.stringify_keys] - end - end - - def import_bookmarks! - parse_import_data!(['#uri']) - items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row['#uri'].strip } - - if @import.overwrite? - presence_hash = items.index_with(true) - - @account.bookmarks.find_each do |bookmark| - if presence_hash[bookmark.status.uri] - items.delete(bookmark.status.uri) - else - bookmark.destroy! - end - end - end - - statuses = items.filter_map do |uri| - status = ActivityPub::TagManager.instance.uri_to_resource(uri, Status) - next if status.nil? && ActivityPub::TagManager.instance.local_uri?(uri) - - status || ActivityPub::FetchRemoteStatusService.new.call(uri) - rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::UnexpectedResponseError - nil - rescue => e - Rails.logger.warn "Unexpected error when importing bookmark: #{e}" - nil - end - - account_ids = statuses.map(&:account_id) - preloaded_relations = @account.relations_map(account_ids, skip_blocking_and_muting: true) - - statuses.keep_if { |status| StatusPolicy.new(@account, status, preloaded_relations).show? } - - statuses.each do |status| - @account.bookmarks.find_or_create_by!(account: @account, status: status) - end - end - - def parse_import_data!(default_headers) - data = CSV.parse(import_data, headers: true) - data = CSV.parse(import_data, headers: default_headers) unless data.headers&.first&.strip&.include?(' ') - @data = data.compact_blank - end - - def import_data - Paperclip.io_adapters.for(@import.data).read.force_encoding(Encoding::UTF_8) - end -end diff --git a/app/views/settings/imports/index.html.haml b/app/views/settings/imports/index.html.haml index ca815720fd6..f451e886062 100644 --- a/app/views/settings/imports/index.html.haml +++ b/app/views/settings/imports/index.html.haml @@ -23,7 +23,7 @@ = f.input :mode, as: :radio_buttons, collection_wrapper_tag: 'ul', - collection: Import::MODES, + collection: Form::Import::MODES, item_wrapper_tag: 'li', label_method: ->(mode) { safe_join([I18n.t("imports.modes.#{mode}"), content_tag(:span, I18n.t("imports.modes.#{mode}_long"), class: 'hint')]) } diff --git a/app/workers/import/relationship_worker.rb b/app/workers/import/relationship_worker.rb deleted file mode 100644 index 2298b095a77..00000000000 --- a/app/workers/import/relationship_worker.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -# NOTE: This is a deprecated worker, only kept to not break ongoing imports -# on upgrade. See `Import::RowWorker` for its replacement. - -class Import::RelationshipWorker - include Sidekiq::Worker - - sidekiq_options queue: 'pull', retry: 8, dead: false - - def perform(account_id, target_account_uri, relationship, options) - from_account = Account.find(account_id) - target_domain = domain(target_account_uri) - target_account = stoplight_wrapper(target_domain).run { ResolveAccountService.new.call(target_account_uri, { check_delivery_availability: true }) } - options.symbolize_keys! - - return if target_account.nil? - - case relationship - when 'follow' - begin - FollowService.new.call(from_account, target_account, **options) - rescue ActiveRecord::RecordInvalid - raise if FollowLimitValidator.limit_for_account(from_account) < from_account.following_count - end - when 'unfollow' - UnfollowService.new.call(from_account, target_account) - when 'block' - BlockService.new.call(from_account, target_account) - when 'unblock' - UnblockService.new.call(from_account, target_account) - when 'mute' - MuteService.new.call(from_account, target_account, **options) - when 'unmute' - UnmuteService.new.call(from_account, target_account) - end - rescue ActiveRecord::RecordNotFound - true - end - - def domain(uri) - domain = uri.is_a?(Account) ? uri.domain : uri.split('@')[1] - TagManager.instance.local_domain?(domain) ? nil : TagManager.instance.normalize_domain(domain) - end - - def stoplight_wrapper(domain) - if domain.present? - Stoplight("source:#{domain}") - .with_fallback { nil } - .with_threshold(1) - .with_cool_off_time(5.minutes.seconds) - .with_error_handler { |error, handle| error.is_a?(HTTP::Error) || error.is_a?(OpenSSL::SSL::SSLError) ? handle.call(error) : raise(error) } - else - Stoplight('domain-blank') - end - end -end diff --git a/app/workers/import_worker.rb b/app/workers/import_worker.rb deleted file mode 100644 index b6afb972a96..00000000000 --- a/app/workers/import_worker.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -# NOTE: This is a deprecated worker, only kept to not break ongoing imports -# on upgrade. See `ImportWorker` for its replacement. - -class ImportWorker - include Sidekiq::Worker - - sidekiq_options queue: 'pull', retry: false - - def perform(import_id) - import = Import.find(import_id) - ImportService.new.call(import) - ensure - import&.destroy - end -end diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb index 6b14985ae30..02ea580b38b 100644 --- a/config/initializers/paperclip.rb +++ b/config/initializers/paperclip.rb @@ -168,7 +168,7 @@ else end Rails.application.reloader.to_prepare do - Paperclip.options[:content_type_mappings] = { csv: Import::FILE_TYPES } + Paperclip.options[:content_type_mappings] = { csv: %w(text/plain text/csv application/csv) } end # In some places in the code, we rescue this exception, but we don't always diff --git a/db/migrate/20240517144908_drop_imports.rb b/db/migrate/20240517144908_drop_imports.rb new file mode 100644 index 00000000000..7be9daf7505 --- /dev/null +++ b/db/migrate/20240517144908_drop_imports.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class DropImports < ActiveRecord::Migration[7.1] + def up + drop_table :imports + end + + def down + raise ActiveRecord::IrreversibleMigration + end +end diff --git a/db/schema.rb b/db/schema.rb index f01e11792de..2b7810da96e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -551,19 +551,6 @@ ActiveRecord::Schema[7.1].define(version: 2024_08_08_125420) do t.index ["user_id"], name: "index_identities_on_user_id" end - create_table "imports", force: :cascade do |t| - t.integer "type", null: false - t.boolean "approved", default: false, null: false - t.datetime "created_at", precision: nil, null: false - t.datetime "updated_at", precision: nil, null: false - t.string "data_file_name" - t.string "data_content_type" - t.integer "data_file_size" - t.datetime "data_updated_at", precision: nil - t.bigint "account_id", null: false - t.boolean "overwrite", default: false, null: false - end - create_table "invites", force: :cascade do |t| t.bigint "user_id", null: false t.string "code", default: "", null: false @@ -1323,7 +1310,6 @@ ActiveRecord::Schema[7.1].define(version: 2024_08_08_125420) do add_foreign_key "follows", "accounts", name: "fk_32ed1b5560", on_delete: :cascade add_foreign_key "generated_annual_reports", "accounts" add_foreign_key "identities", "users", name: "fk_bea040f377", on_delete: :cascade - add_foreign_key "imports", "accounts", name: "fk_6db1b6e408", on_delete: :cascade add_foreign_key "invites", "users", on_delete: :cascade add_foreign_key "list_accounts", "accounts", on_delete: :cascade add_foreign_key "list_accounts", "follow_requests", on_delete: :cascade diff --git a/lib/mastodon/cli/media.rb b/lib/mastodon/cli/media.rb index 123973d1954..ec0c587e138 100644 --- a/lib/mastodon/cli/media.rb +++ b/lib/mastodon/cli/media.rb @@ -284,7 +284,6 @@ module Mastodon::CLI say("Avatars:\t#{number_to_human_size(Account.sum(:avatar_file_size))} (#{number_to_human_size(Account.local.sum(:avatar_file_size))} local)") say("Headers:\t#{number_to_human_size(Account.sum(:header_file_size))} (#{number_to_human_size(Account.local.sum(:header_file_size))} local)") say("Backups:\t#{number_to_human_size(Backup.sum(:dump_file_size))}") - say("Imports:\t#{number_to_human_size(Import.sum(:data_file_size))}") say("Settings:\t#{number_to_human_size(SiteUpload.sum(:file_file_size))}") end @@ -338,7 +337,6 @@ module Mastodon::CLI Account Backup CustomEmoji - Import MediaAttachment PreviewCard SiteUpload diff --git a/spec/fabricators/import_fabricator.rb b/spec/fabricators/import_fabricator.rb deleted file mode 100644 index 4951bb9a4da..00000000000 --- a/spec/fabricators/import_fabricator.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -Fabricator(:import) do - account - type :following - data { attachment_fixture('imports.txt') } -end diff --git a/spec/models/import_spec.rb b/spec/models/import_spec.rb deleted file mode 100644 index 10df5f8c0b2..00000000000 --- a/spec/models/import_spec.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -RSpec.describe Import do - let(:account) { Fabricate(:account) } - let(:type) { 'following' } - let(:data) { attachment_fixture('imports.txt') } - - describe 'validations' do - it 'is invalid without an type' do - import = described_class.create(account: account, data: data) - expect(import).to model_have_error_on_field(:type) - end - - it 'is invalid without a data' do - import = described_class.create(account: account, type: type) - expect(import).to model_have_error_on_field(:data) - end - end -end diff --git a/spec/services/import_service_spec.rb b/spec/services/import_service_spec.rb deleted file mode 100644 index 0a99c5e748d..00000000000 --- a/spec/services/import_service_spec.rb +++ /dev/null @@ -1,242 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -RSpec.describe ImportService, :inline_jobs do - include RoutingHelper - - let!(:account) { Fabricate(:account, locked: false) } - let!(:bob) { Fabricate(:account, username: 'bob', locked: false) } - let!(:eve) { Fabricate(:account, username: 'eve', domain: 'example.com', locked: false, protocol: :activitypub, inbox_url: 'https://example.com/inbox') } - - before do - stub_request(:post, 'https://example.com/inbox').to_return(status: 200) - end - - context 'when importing old-style list of muted users' do - subject { described_class.new } - - let(:csv) { attachment_fixture('mute-imports.txt') } - - describe 'when no accounts are muted' do - let(:import) { Import.create(account: account, type: 'muting', data: csv) } - - it 'mutes the listed accounts, including notifications' do - subject.call(import) - expect(account.muting.count).to eq 2 - expect(Mute.find_by(account: account, target_account: bob).hide_notifications).to be true - end - end - - describe 'when some accounts are muted and overwrite is not set' do - let(:import) { Import.create(account: account, type: 'muting', data: csv) } - - it 'mutes the listed accounts, including notifications' do - account.mute!(bob, notifications: false) - subject.call(import) - expect(account.muting.count).to eq 2 - expect(Mute.find_by(account: account, target_account: bob).hide_notifications).to be true - end - end - - describe 'when some accounts are muted and overwrite is set' do - let(:import) { Import.create(account: account, type: 'muting', data: csv, overwrite: true) } - - it 'mutes the listed accounts, including notifications' do - account.mute!(bob, notifications: false) - subject.call(import) - expect(account.muting.count).to eq 2 - expect(Mute.find_by(account: account, target_account: bob).hide_notifications).to be true - end - end - end - - context 'when importing new-style list of muted users' do - subject { described_class.new } - - let(:csv) { attachment_fixture('new-mute-imports.txt') } - - describe 'when no accounts are muted' do - let(:import) { Import.create(account: account, type: 'muting', data: csv) } - - it 'mutes the listed accounts, respecting notifications' do - subject.call(import) - expect(account.muting.count).to eq 2 - expect(Mute.find_by(account: account, target_account: bob).hide_notifications).to be true - expect(Mute.find_by(account: account, target_account: eve).hide_notifications).to be false - end - end - - describe 'when some accounts are muted and overwrite is not set' do - let(:import) { Import.create(account: account, type: 'muting', data: csv) } - - it 'mutes the listed accounts, respecting notifications' do - account.mute!(bob, notifications: true) - subject.call(import) - expect(account.muting.count).to eq 2 - expect(Mute.find_by(account: account, target_account: bob).hide_notifications).to be true - expect(Mute.find_by(account: account, target_account: eve).hide_notifications).to be false - end - end - - describe 'when some accounts are muted and overwrite is set' do - let(:import) { Import.create(account: account, type: 'muting', data: csv, overwrite: true) } - - it 'mutes the listed accounts, respecting notifications' do - account.mute!(bob, notifications: true) - subject.call(import) - expect(account.muting.count).to eq 2 - expect(Mute.find_by(account: account, target_account: bob).hide_notifications).to be true - expect(Mute.find_by(account: account, target_account: eve).hide_notifications).to be false - end - end - end - - context 'when importing old-style list of followed users' do - subject { described_class.new } - - let(:csv) { attachment_fixture('mute-imports.txt') } - - describe 'when no accounts are followed' do - let(:import) { Import.create(account: account, type: 'following', data: csv) } - - it 'follows the listed accounts, including boosts' do - subject.call(import) - - expect(account.following.count).to eq 1 - expect(account.follow_requests.count).to eq 1 - expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true - end - end - - describe 'when some accounts are already followed and overwrite is not set' do - let(:import) { Import.create(account: account, type: 'following', data: csv) } - - it 'follows the listed accounts, including notifications' do - account.follow!(bob, reblogs: false) - subject.call(import) - expect(account.following.count).to eq 1 - expect(account.follow_requests.count).to eq 1 - expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true - end - end - - describe 'when some accounts are already followed and overwrite is set' do - let(:import) { Import.create(account: account, type: 'following', data: csv, overwrite: true) } - - it 'mutes the listed accounts, including notifications' do - account.follow!(bob, reblogs: false) - subject.call(import) - expect(account.following.count).to eq 1 - expect(account.follow_requests.count).to eq 1 - expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true - end - end - end - - context 'when importing new-style list of followed users' do - subject { described_class.new } - - let(:csv) { attachment_fixture('new-following-imports.txt') } - - describe 'when no accounts are followed' do - let(:import) { Import.create(account: account, type: 'following', data: csv) } - - it 'follows the listed accounts, respecting boosts' do - subject.call(import) - expect(account.following.count).to eq 1 - expect(account.follow_requests.count).to eq 1 - expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true - expect(FollowRequest.find_by(account: account, target_account: eve).show_reblogs).to be false - end - end - - describe 'when some accounts are already followed and overwrite is not set' do - let(:import) { Import.create(account: account, type: 'following', data: csv) } - - it 'mutes the listed accounts, respecting notifications' do - account.follow!(bob, reblogs: true) - subject.call(import) - expect(account.following.count).to eq 1 - expect(account.follow_requests.count).to eq 1 - expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true - expect(FollowRequest.find_by(account: account, target_account: eve).show_reblogs).to be false - end - end - - describe 'when some accounts are already followed and overwrite is set' do - let(:import) { Import.create(account: account, type: 'following', data: csv, overwrite: true) } - - it 'mutes the listed accounts, respecting notifications' do - account.follow!(bob, reblogs: true) - subject.call(import) - expect(account.following.count).to eq 1 - expect(account.follow_requests.count).to eq 1 - expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true - expect(FollowRequest.find_by(account: account, target_account: eve).show_reblogs).to be false - end - end - end - - # Based on the bug report 20571 where UTF-8 encoded domains were rejecting import of their users - # - # https://github.com/mastodon/mastodon/issues/20571 - context 'with a utf-8 encoded domains' do - subject { described_class.new } - - let!(:nare) { Fabricate(:account, username: 'nare', domain: 'թութ.հայ', locked: false, protocol: :activitypub, inbox_url: 'https://թութ.հայ/inbox') } - let(:csv) { attachment_fixture('utf8-followers.txt') } - let(:import) { Import.create(account: account, type: 'following', data: csv) } - - # Make sure to not actually go to the remote server - before do - stub_request(:post, nare.inbox_url).to_return(status: 200) - end - - it 'follows the listed account' do - expect(account.follow_requests.count).to eq 0 - subject.call(import) - expect(account.follow_requests.count).to eq 1 - end - end - - context 'when importing bookmarks' do - subject { described_class.new } - - let(:csv) { attachment_fixture('bookmark-imports.txt') } - let(:local_account) { Fabricate(:account, username: 'foo', domain: '') } - let!(:remote_status) { Fabricate(:status, uri: 'https://example.com/statuses/1312') } - let!(:direct_status) { Fabricate(:status, uri: 'https://example.com/statuses/direct', visibility: :direct) } - - around do |example| - local_before = Rails.configuration.x.local_domain - web_before = Rails.configuration.x.web_domain - Rails.configuration.x.local_domain = 'local.com' - Rails.configuration.x.web_domain = 'local.com' - example.run - Rails.configuration.x.web_domain = web_before - Rails.configuration.x.local_domain = local_before - end - - before do - service = instance_double(ActivityPub::FetchRemoteStatusService) - allow(ActivityPub::FetchRemoteStatusService).to receive(:new).and_return(service) - allow(service).to receive(:call).with('https://unknown-remote.com/users/bar/statuses/1') do - Fabricate(:status, uri: 'https://unknown-remote.com/users/bar/statuses/1') - end - end - - describe 'when no bookmarks are set' do - let(:import) { Import.create(account: account, type: 'bookmarks', data: csv) } - - it 'adds the toots the user has access to to bookmarks' do - local_status = Fabricate(:status, account: local_account, uri: 'https://local.com/users/foo/statuses/42', id: 42, local: true) - subject.call(import) - expect(account.bookmarks.map { |bookmark| bookmark.status.id }).to include(local_status.id) - expect(account.bookmarks.map { |bookmark| bookmark.status.id }).to include(remote_status.id) - expect(account.bookmarks.map { |bookmark| bookmark.status.id }).to_not include(direct_status.id) - expect(account.bookmarks.count).to eq 3 - end - end - end -end diff --git a/spec/workers/import_worker_spec.rb b/spec/workers/import_worker_spec.rb deleted file mode 100644 index 4095a5d354b..00000000000 --- a/spec/workers/import_worker_spec.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -describe ImportWorker do - let(:worker) { described_class.new } - let(:service) { instance_double(ImportService, call: true) } - - describe '#perform' do - before do - allow(ImportService).to receive(:new).and_return(service) - end - - let(:import) { Fabricate(:import) } - - it 'sends the import to the service' do - worker.perform(import.id) - - expect(service).to have_received(:call).with(import) - expect { import.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end -end