From ed8e4bab4c8232533bd40d36c2ea5524e078f0af Mon Sep 17 00:00:00 2001 From: Matt Jankowski Date: Thu, 14 Dec 2023 09:02:33 -0500 Subject: [PATCH 01/27] Fix reference to non-existent var in CLI maintenance command (#28363) --- lib/mastodon/cli/maintenance.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mastodon/cli/maintenance.rb b/lib/mastodon/cli/maintenance.rb index e73bcbf86..c2a6802e1 100644 --- a/lib/mastodon/cli/maintenance.rb +++ b/lib/mastodon/cli/maintenance.rb @@ -224,7 +224,7 @@ module Mastodon::CLI users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse ref_user = users.shift say "Multiple users registered with e-mail address #{ref_user.email}.", :yellow - say "e-mail will be disabled for the following accounts: #{user.map(&:account).map(&:acct).join(', ')}", :yellow + say "e-mail will be disabled for the following accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow say 'Please reach out to them and set another address with `tootctl account modify` or delete them.', :yellow users.each_with_index do |user, index| From 6536d96d1b211cb4b462010474cb63b241de9998 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 4 Jan 2024 15:14:46 +0100 Subject: [PATCH 02/27] Add fallback redirection when getting a webfinger query `WEB_DOMAIN@WEB_DOMAIN` (#28592) --- app/controllers/well_known/webfinger_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/well_known/webfinger_controller.rb b/app/controllers/well_known/webfinger_controller.rb index 4748940f7..6cf37c2ff 100644 --- a/app/controllers/well_known/webfinger_controller.rb +++ b/app/controllers/well_known/webfinger_controller.rb @@ -21,7 +21,7 @@ module WellKnown username = username_from_resource @account = begin - if username == Rails.configuration.x.local_domain + if username == Rails.configuration.x.local_domain || username == Rails.configuration.x.web_domain Account.representative else Account.find_local!(username) From f784213c6405cbca2f3ff70479aff2e263b28730 Mon Sep 17 00:00:00 2001 From: Emelia Smith Date: Tue, 6 Feb 2024 13:38:14 +0100 Subject: [PATCH 03/27] Return domain block digests from admin domain blocks API (#29092) --- app/serializers/rest/admin/domain_block_serializer.rb | 6 +++++- spec/requests/api/v1/admin/domain_blocks_spec.rb | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/app/serializers/rest/admin/domain_block_serializer.rb b/app/serializers/rest/admin/domain_block_serializer.rb index b955d008a..e94a337cb 100644 --- a/app/serializers/rest/admin/domain_block_serializer.rb +++ b/app/serializers/rest/admin/domain_block_serializer.rb @@ -1,11 +1,15 @@ # frozen_string_literal: true class REST::Admin::DomainBlockSerializer < ActiveModel::Serializer - attributes :id, :domain, :created_at, :severity, + attributes :id, :domain, :digest, :created_at, :severity, :reject_media, :reject_reports, :private_comment, :public_comment, :obfuscate def id object.id.to_s end + + def digest + object.domain_digest + end end diff --git a/spec/requests/api/v1/admin/domain_blocks_spec.rb b/spec/requests/api/v1/admin/domain_blocks_spec.rb index 7a5ac28c5..a6cfdb48d 100644 --- a/spec/requests/api/v1/admin/domain_blocks_spec.rb +++ b/spec/requests/api/v1/admin/domain_blocks_spec.rb @@ -49,6 +49,7 @@ RSpec.describe 'Domain Blocks' do { id: domain_block.id.to_s, domain: domain_block.domain, + digest: domain_block.domain_digest, created_at: domain_block.created_at.strftime('%Y-%m-%dT%H:%M:%S.%LZ'), severity: domain_block.severity.to_s, reject_media: domain_block.reject_media, @@ -102,6 +103,7 @@ RSpec.describe 'Domain Blocks' do { id: domain_block.id.to_s, domain: domain_block.domain, + digest: domain_block.domain_digest, created_at: domain_block.created_at.strftime('%Y-%m-%dT%H:%M:%S.%LZ'), severity: domain_block.severity.to_s, reject_media: domain_block.reject_media, @@ -212,6 +214,7 @@ RSpec.describe 'Domain Blocks' do { id: domain_block.id.to_s, domain: domain_block.domain, + digest: domain_block.domain_digest, severity: 'suspend', } ) From 7af69f5cf575b1124a78c0c8e5b24ebce8060621 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 23 Feb 2024 20:04:57 +0100 Subject: [PATCH 04/27] Fix admin account created by `mastodon:setup` not being auto-approved (#29379) --- lib/tasks/mastodon.rake | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index f68d1cf1f..8e024077e 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -515,6 +515,7 @@ namespace :mastodon do owner_role = UserRole.find_by(name: 'Owner') user = User.new(email: email, password: password, confirmed_at: Time.now.utc, account_attributes: { username: username }, bypass_invite_request_check: true, role: owner_role) user.save(validate: false) + user.approve! Setting.site_contact_username = username From ab3f9852f23a2180ae77b3a0963fe61e97ab129f Mon Sep 17 00:00:00 2001 From: Jeong Arm Date: Mon, 11 Mar 2024 18:28:08 +0900 Subject: [PATCH 05/27] Normalize idna domain before account unblock domain (#29530) --- app/models/concerns/account_interactions.rb | 6 ++- .../concerns/account_interactions_spec.rb | 40 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb index 2de15031f..74ba1695f 100644 --- a/app/models/concerns/account_interactions.rb +++ b/app/models/concerns/account_interactions.rb @@ -185,7 +185,7 @@ module AccountInteractions end def unblock_domain!(other_domain) - block = domain_blocks.find_by(domain: other_domain) + block = domain_blocks.find_by(domain: normalized_domain(other_domain)) block&.destroy end @@ -313,4 +313,8 @@ module AccountInteractions def remove_potential_friendship(other_account) PotentialFriendshipTracker.remove(id, other_account.id) end + + def normalized_domain(domain) + TagManager.instance.normalize_domain(domain) + end end diff --git a/spec/models/concerns/account_interactions_spec.rb b/spec/models/concerns/account_interactions_spec.rb index 84e2c91a8..64d1d46d3 100644 --- a/spec/models/concerns/account_interactions_spec.rb +++ b/spec/models/concerns/account_interactions_spec.rb @@ -250,6 +250,24 @@ describe AccountInteractions do end end + describe '#block_idna_domain!' do + subject do + [ + account.block_domain!(idna_domain), + account.block_domain!(punycode_domain), + ] + end + + let(:idna_domain) { '대한민국.한국' } + let(:punycode_domain) { 'xn--3e0bs9hfvinn1a.xn--3e0b707e' } + + it 'creates single AccountDomainBlock' do + expect do + expect(subject).to all(be_a AccountDomainBlock) + end.to change { account.domain_blocks.count }.by 1 + end + end + describe '#unfollow!' do subject { account.unfollow!(target_account) } @@ -345,6 +363,28 @@ describe AccountInteractions do end end + describe '#unblock_idna_domain!' do + subject { account.unblock_domain!(punycode_domain) } + + let(:idna_domain) { '대한민국.한국' } + let(:punycode_domain) { 'xn--3e0bs9hfvinn1a.xn--3e0b707e' } + + context 'when blocking the domain' do + it 'returns destroyed AccountDomainBlock' do + account_domain_block = Fabricate(:account_domain_block, domain: idna_domain) + account.domain_blocks << account_domain_block + expect(subject).to be_a AccountDomainBlock + expect(subject).to be_destroyed + end + end + + context 'when unblocking idna domain' do + it 'returns nil' do + expect(subject).to be_nil + end + end + end + describe '#following?' do subject { account.following?(target_account) } From 0143c9d3e1d7ae11b14aff5a0326885cbbd3ed19 Mon Sep 17 00:00:00 2001 From: Matt Jankowski Date: Fri, 22 Mar 2024 11:08:27 -0400 Subject: [PATCH 06/27] Fix results/query in `api/v1/featured_tags/suggestions` (#29597) --- .../featured_tags/suggestions_controller.rb | 6 +++- .../suggestions_controller_spec.rb | 28 +++++++++++++++++-- spec/fabricators/featured_tag_fabricator.rb | 2 +- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/v1/featured_tags/suggestions_controller.rb b/app/controllers/api/v1/featured_tags/suggestions_controller.rb index 76633210a..4f732ed2d 100644 --- a/app/controllers/api/v1/featured_tags/suggestions_controller.rb +++ b/app/controllers/api/v1/featured_tags/suggestions_controller.rb @@ -12,6 +12,10 @@ class Api::V1::FeaturedTags::SuggestionsController < Api::BaseController private def set_recently_used_tags - @recently_used_tags = Tag.recently_used(current_account).where.not(id: current_account.featured_tags).limit(10) + @recently_used_tags = Tag.recently_used(current_account).where.not(id: featured_tag_ids).limit(10) + end + + def featured_tag_ids + current_account.featured_tags.pluck(:tag_id) end end diff --git a/spec/controllers/api/v1/featured_tags/suggestions_controller_spec.rb b/spec/controllers/api/v1/featured_tags/suggestions_controller_spec.rb index 54c63dcc6..8cb928ea2 100644 --- a/spec/controllers/api/v1/featured_tags/suggestions_controller_spec.rb +++ b/spec/controllers/api/v1/featured_tags/suggestions_controller_spec.rb @@ -7,17 +7,39 @@ describe Api::V1::FeaturedTags::SuggestionsController do let(:user) { Fabricate(:user) } let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:accounts') } - let(:account) { Fabricate(:account) } + let(:account) { Fabricate(:account, user: user) } before do allow(controller).to receive(:doorkeeper_token) { token } end describe 'GET #index' do - it 'returns http success' do + let!(:unused_featured_tag) { Fabricate(:tag, name: 'unused_featured_tag') } + let!(:used_tag) { Fabricate(:tag, name: 'used_tag') } + let!(:used_featured_tag) { Fabricate(:tag, name: 'used_featured_tag') } + + before do + _unused_tag = Fabricate(:tag, name: 'unused_tag') + + # Make relevant tags used by account + status = Fabricate(:status, account: account) + status.tags << used_tag + status.tags << used_featured_tag + + # Feature the relevant tags + Fabricate :featured_tag, account: account, name: unused_featured_tag.name + Fabricate :featured_tag, account: account, name: used_featured_tag.name + end + + it 'returns http success and recently used but not featured tags', :aggregate_failures do get :index, params: { account_id: account.id, limit: 2 } - expect(response).to have_http_status(200) + expect(response) + .to have_http_status(200) + expect(body_as_json) + .to contain_exactly( + include(name: used_tag.name) + ) end end end diff --git a/spec/fabricators/featured_tag_fabricator.rb b/spec/fabricators/featured_tag_fabricator.rb index 0803dc43a..6003099db 100644 --- a/spec/fabricators/featured_tag_fabricator.rb +++ b/spec/fabricators/featured_tag_fabricator.rb @@ -2,6 +2,6 @@ Fabricator(:featured_tag) do account { Fabricate.build(:account) } - tag { Fabricate.build(:tag) } + tag { nil } name { sequence(:name) { |i| "Tag#{i}" } } end From 86807e4799d21863084d52d4070f3da7c120e260 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 5 Apr 2024 09:48:45 +0200 Subject: [PATCH 07/27] Improve email address validation (#29838) --- Gemfile | 2 ++ Gemfile.lock | 1 + app/models/user.rb | 2 ++ app/validators/email_address_validator.rb | 18 ++++++++++++++++++ spec/models/user_spec.rb | 6 ++++++ 5 files changed, 29 insertions(+) create mode 100644 app/validators/email_address_validator.rb diff --git a/Gemfile b/Gemfile index 829e7d857..1d038ad9f 100644 --- a/Gemfile +++ b/Gemfile @@ -204,3 +204,5 @@ gem 'net-http', '~> 0.3.2' gem 'rubyzip', '~> 2.3' gem 'hcaptcha', '~> 7.1' + +gem 'mail', '~> 2.8' diff --git a/Gemfile.lock b/Gemfile.lock index 9290a84ae..94d2d0ba4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -871,6 +871,7 @@ DEPENDENCIES letter_opener_web (~> 2.0) link_header (~> 0.0) lograge (~> 0.12) + mail (~> 2.8) mario-redis-lock (~> 1.2) md-paperclip-azure (~> 2.2) memory_profiler diff --git a/app/models/user.rb b/app/models/user.rb index ee9097927..0c8d481c4 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -96,6 +96,8 @@ class User < ApplicationRecord accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? && !Setting.require_invite_text } validates :invite_request, presence: true, on: :create, if: :invite_text_required? + validates :email, presence: true, email_address: true + validates_with BlacklistedEmailValidator, if: -> { ENV['EMAIL_DOMAIN_LISTS_APPLY_AFTER_CONFIRMATION'] == 'true' || !confirmed? } validates_with EmailMxValidator, if: :validate_email_dns? validates :agreement, acceptance: { allow_nil: false, accept: [true, 'true', '1'] }, on: :create diff --git a/app/validators/email_address_validator.rb b/app/validators/email_address_validator.rb new file mode 100644 index 000000000..ed0bb1165 --- /dev/null +++ b/app/validators/email_address_validator.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# NOTE: I initially wrote this as `EmailValidator` but it ended up clashing +# with an indirect dependency of ours, `validate_email`, which, turns out, +# has the same approach as we do, but with an extra check disallowing +# single-label domains. Decided to not switch to `validate_email` because +# we do want to allow at least `localhost`. + +class EmailAddressValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + value = value.strip + + address = Mail::Address.new(value) + record.errors.add(attribute, :invalid) if address.address != value + rescue Mail::Field::FieldError + record.errors.add(attribute, :invalid) + end +end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index ffd1889cb..f06150f02 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -39,6 +39,12 @@ RSpec.describe User do expect(user.valid?).to be true end + it 'is valid with a localhost e-mail address' do + user = Fabricate.build(:user, email: 'admin@localhost') + user.valid? + expect(user.valid?).to be true + end + it 'cleans out invalid locale' do user = Fabricate.build(:user, locale: 'toto') expect(user.valid?).to be true From c3be5a3d2eff947a04a32d78726ff511d7dec98f Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 8 Apr 2024 15:46:13 +0200 Subject: [PATCH 08/27] Remove caching in `cache_collection` (#29862) --- app/controllers/concerns/cache_concern.rb | 29 ++++----------- app/models/concerns/cacheable.rb | 4 +++ app/models/feed.rb | 2 +- app/models/public_feed.rb | 2 +- app/models/status.rb | 32 ----------------- app/models/tag_feed.rb | 2 +- .../application_controller_spec.rb | 35 ------------------- spec/models/home_feed_spec.rb | 1 - 8 files changed, 14 insertions(+), 93 deletions(-) diff --git a/app/controllers/concerns/cache_concern.rb b/app/controllers/concerns/cache_concern.rb index 55ebe1bd6..c8ed9f9a5 100644 --- a/app/controllers/concerns/cache_concern.rb +++ b/app/controllers/concerns/cache_concern.rb @@ -198,34 +198,19 @@ module CacheConcern end end + # TODO: Rename this method, as it does not perform any caching anymore. def cache_collection(raw, klass) - return raw unless klass.respond_to?(:with_includes) + return raw unless klass.respond_to?(:preload_cacheable_associations) - raw = raw.cache_ids.to_a if raw.is_a?(ActiveRecord::Relation) - return [] if raw.empty? + records = raw.to_a - cached_keys_with_value = begin - Rails.cache.read_multi(*raw).transform_keys(&:id).transform_values { |r| ActiveRecordCoder.load(r) } - rescue ActiveRecordCoder::Error - {} # The serialization format may have changed, let's pretend it's a cache miss. - end + klass.preload_cacheable_associations(records) - uncached_ids = raw.map(&:id) - cached_keys_with_value.keys - - klass.reload_stale_associations!(cached_keys_with_value.values) if klass.respond_to?(:reload_stale_associations!) - - unless uncached_ids.empty? - uncached = klass.where(id: uncached_ids).with_includes.index_by(&:id) - - uncached.each_value do |item| - Rails.cache.write(item, ActiveRecordCoder.dump(item)) - end - end - - raw.filter_map { |item| cached_keys_with_value[item.id] || uncached[item.id] } + records end + # TODO: Rename this method, as it does not perform any caching anymore. def cache_collection_paginated_by_id(raw, klass, limit, options) - cache_collection raw.cache_ids.to_a_paginated_by_id(limit, options), klass + cache_collection raw.to_a_paginated_by_id(limit, options), klass end end diff --git a/app/models/concerns/cacheable.rb b/app/models/concerns/cacheable.rb index d7524cdfd..0633f20c7 100644 --- a/app/models/concerns/cacheable.rb +++ b/app/models/concerns/cacheable.rb @@ -14,6 +14,10 @@ module Cacheable includes(@cache_associated) end + def preload_cacheable_associations(records) + ActiveRecord::Associations::Preloader.new(records: records, associations: @cache_associated).call + end + def cache_ids select(:id, :updated_at) end diff --git a/app/models/feed.rb b/app/models/feed.rb index f51dcfab1..30073fed4 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -28,7 +28,7 @@ class Feed unhydrated = redis.zrangebyscore(key, "(#{min_id}", "(#{max_id}", limit: [0, limit], with_scores: true).map(&:first).map(&:to_i) end - Status.where(id: unhydrated).cache_ids + Status.where(id: unhydrated) end def key diff --git a/app/models/public_feed.rb b/app/models/public_feed.rb index c208d6f66..df59c9c22 100644 --- a/app/models/public_feed.rb +++ b/app/models/public_feed.rb @@ -29,7 +29,7 @@ class PublicFeed scope.merge!(media_only_scope) if media_only? scope.merge!(language_scope) if account&.chosen_languages.present? - scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id) + scope.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id) end private diff --git a/app/models/status.rb b/app/models/status.rb index 1c41ef1d5..990ac8061 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -338,38 +338,6 @@ class Status < ApplicationRecord StatusPin.select('status_id').where(status_id: status_ids).where(account_id: account_id).each_with_object({}) { |p, h| h[p.status_id] = true } end - def reload_stale_associations!(cached_items) - account_ids = [] - - cached_items.each do |item| - account_ids << item.account_id - account_ids << item.reblog.account_id if item.reblog? - end - - account_ids.uniq! - - status_ids = cached_items.map { |item| item.reblog? ? item.reblog_of_id : item.id }.uniq - - return if account_ids.empty? - - accounts = Account.where(id: account_ids).includes(:account_stat, :user).index_by(&:id) - - status_stats = StatusStat.where(status_id: status_ids).index_by(&:status_id) - - cached_items.each do |item| - item.account = accounts[item.account_id] - item.reblog.account = accounts[item.reblog.account_id] if item.reblog? - - if item.reblog? - status_stat = status_stats[item.reblog.id] - item.reblog.status_stat = status_stat if status_stat.present? - else - status_stat = status_stats[item.id] - item.status_stat = status_stat if status_stat.present? - end - end - end - def from_text(text) return [] if text.blank? diff --git a/app/models/tag_feed.rb b/app/models/tag_feed.rb index b8cd63557..6b5831d24 100644 --- a/app/models/tag_feed.rb +++ b/app/models/tag_feed.rb @@ -33,7 +33,7 @@ class TagFeed < PublicFeed scope.merge!(account_filters_scope) if account? scope.merge!(media_only_scope) if media_only? - scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id) + scope.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id) end private diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb index e9d479603..5cd45e991 100644 --- a/spec/controllers/application_controller_spec.rb +++ b/spec/controllers/application_controller_spec.rb @@ -221,39 +221,4 @@ describe ApplicationController do include_examples 'respond_with_error', 422 end - - describe 'cache_collection' do - subject do - Class.new(ApplicationController) do - public :cache_collection - end - end - - shared_examples 'receives :with_includes' do |fabricator, klass| - it 'uses raw if it is not an ActiveRecord::Relation' do - record = Fabricate(fabricator) - expect(subject.new.cache_collection([record], klass)).to eq [record] - end - end - - shared_examples 'cacheable' do |fabricator, klass| - include_examples 'receives :with_includes', fabricator, klass - - it 'calls cache_ids of raw if it is an ActiveRecord::Relation' do - record = Fabricate(fabricator) - relation = klass.none - allow(relation).to receive(:cache_ids).and_return([record]) - expect(subject.new.cache_collection(relation, klass)).to eq [record] - end - end - - it 'returns raw unless class responds to :with_includes' do - raw = Object.new - expect(subject.new.cache_collection(raw, Object)).to eq raw - end - - context 'with a Status' do - include_examples 'cacheable', :status, Status - end - end end diff --git a/spec/models/home_feed_spec.rb b/spec/models/home_feed_spec.rb index bd649d826..06bb63b1a 100644 --- a/spec/models/home_feed_spec.rb +++ b/spec/models/home_feed_spec.rb @@ -27,7 +27,6 @@ RSpec.describe HomeFeed do results = subject.get(3) expect(results.map(&:id)).to eq [3, 2] - expect(results.first.attributes.keys).to eq %w(id updated_at) end end From e69780ec59bbd04a649abc67cc41a824a635fc7b Mon Sep 17 00:00:00 2001 From: Tim Rogers Date: Mon, 22 Apr 2024 04:00:24 -0500 Subject: [PATCH 09/27] Fixed crash when supplying FFMPEG_BINARY environment variable (#30022) --- app/lib/video_metadata_extractor.rb | 2 +- config/initializers/ffmpeg.rb | 5 +++-- lib/paperclip/image_extractor.rb | 2 +- lib/paperclip/transcoder.rb | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/lib/video_metadata_extractor.rb b/app/lib/video_metadata_extractor.rb index f27d34868..df5409375 100644 --- a/app/lib/video_metadata_extractor.rb +++ b/app/lib/video_metadata_extractor.rb @@ -22,7 +22,7 @@ class VideoMetadataExtractor private def ffmpeg_command_output - command = Terrapin::CommandLine.new('ffprobe', '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel') + command = Terrapin::CommandLine.new(Rails.configuration.x.ffprobe_binary, '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel') command.run(path: @path, format: 'json', loglevel: 'fatal') end diff --git a/config/initializers/ffmpeg.rb b/config/initializers/ffmpeg.rb index 30ea617fc..87f85eeec 100644 --- a/config/initializers/ffmpeg.rb +++ b/config/initializers/ffmpeg.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true -if ENV['FFMPEG_BINARY'].present? - FFMPEG.ffmpeg_binary = ENV['FFMPEG_BINARY'] +Rails.application.configure do + config.x.ffmpeg_binary = ENV['FFMPEG_BINARY'] || 'ffmpeg' + config.x.ffprobe_binary = ENV['FFPROBE_BINARY'] || 'ffprobe' end diff --git a/lib/paperclip/image_extractor.rb b/lib/paperclip/image_extractor.rb index 17fe4326f..8a565d046 100644 --- a/lib/paperclip/image_extractor.rb +++ b/lib/paperclip/image_extractor.rb @@ -35,7 +35,7 @@ module Paperclip dst.binmode begin - command = Terrapin::CommandLine.new('ffmpeg', '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger) + command = Terrapin::CommandLine.new(Rails.configuration.x.ffmpeg_binary, '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger) command.run(source: @file.path, destination: dst.path, loglevel: 'fatal') rescue Terrapin::ExitStatusError dst.close(true) diff --git a/lib/paperclip/transcoder.rb b/lib/paperclip/transcoder.rb index d2d946d3a..3efffa355 100644 --- a/lib/paperclip/transcoder.rb +++ b/lib/paperclip/transcoder.rb @@ -61,7 +61,7 @@ module Paperclip command_arguments, interpolations = prepare_command(destination) begin - command = Terrapin::CommandLine.new('ffmpeg', command_arguments.join(' '), logger: Paperclip.logger) + command = Terrapin::CommandLine.new(Rails.configuration.x.ffmpeg_binary, command_arguments.join(' '), logger: Paperclip.logger) command.run(interpolations) rescue Terrapin::ExitStatusError => e raise Paperclip::Error, "Error while transcoding #{@basename}: #{e}" From 51ef619140fc87c23352aced941e313e8aa1c699 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 26 Apr 2024 15:19:02 +0200 Subject: [PATCH 10/27] Fix Idempotency-Key ignored when scheduling a post (#30084) --- app/services/post_status_service.rb | 4 ++-- spec/services/post_status_service_spec.rb | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index ea27f374e..e4dd480f1 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -160,7 +160,7 @@ class PostStatusService < BaseService def idempotency_duplicate if scheduled? - @account.schedule_statuses.find(@idempotency_duplicate) + @account.scheduled_statuses.find(@idempotency_duplicate) else @account.statuses.find(@idempotency_duplicate) end @@ -214,7 +214,7 @@ class PostStatusService < BaseService end def scheduled_options - @options.tap do |options_hash| + @options.dup.tap do |options_hash| options_hash[:in_reply_to_id] = options_hash.delete(:thread)&.id options_hash[:application_id] = options_hash.delete(:application)&.id options_hash[:scheduled_at] = nil diff --git a/spec/services/post_status_service_spec.rb b/spec/services/post_status_service_spec.rb index 7d7679c88..a74e82626 100644 --- a/spec/services/post_status_service_spec.rb +++ b/spec/services/post_status_service_spec.rb @@ -54,6 +54,13 @@ RSpec.describe PostStatusService, type: :service do it 'does not change statuses count' do expect { subject.call(account, text: 'Hi future!', scheduled_at: future, thread: previous_status) }.to_not(change { [account.statuses_count, previous_status.replies_count] }) end + + it 'returns existing status when used twice with idempotency key' do + account = Fabricate(:account) + status1 = subject.call(account, text: 'test', idempotency: 'meepmeep', scheduled_at: future) + status2 = subject.call(account, text: 'test', idempotency: 'meepmeep', scheduled_at: future) + expect(status2.id).to eq status1.id + end end it 'creates response to the original status of boost' do From 56b7d1a7b6c3beaacbcea21a849894d0a575cb13 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 2 May 2024 22:56:21 +0200 Subject: [PATCH 11/27] Fix not being able to block a subdomain of an already-blocked domain through the API (#30119) --- .../api/v1/admin/domain_blocks_controller.rb | 9 +++- .../api/v1/admin/domain_blocks_spec.rb | 45 ++++++++++++++++--- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/v1/admin/domain_blocks_controller.rb b/app/controllers/api/v1/admin/domain_blocks_controller.rb index 2538c7c7c..8786431d9 100644 --- a/app/controllers/api/v1/admin/domain_blocks_controller.rb +++ b/app/controllers/api/v1/admin/domain_blocks_controller.rb @@ -29,10 +29,11 @@ class Api::V1::Admin::DomainBlocksController < Api::BaseController def create authorize :domain_block, :create? + @domain_block = DomainBlock.new(resource_params) existing_domain_block = resource_params[:domain].present? ? DomainBlock.rule_for(resource_params[:domain]) : nil - return render json: existing_domain_block, serializer: REST::Admin::ExistingDomainBlockErrorSerializer, status: 422 if existing_domain_block.present? + return render json: existing_domain_block, serializer: REST::Admin::ExistingDomainBlockErrorSerializer, status: 422 if conflicts_with_existing_block?(@domain_block, existing_domain_block) - @domain_block = DomainBlock.create!(resource_params) + @domain_block.save! DomainBlockWorker.perform_async(@domain_block.id) log_action :create, @domain_block render json: @domain_block, serializer: REST::Admin::DomainBlockSerializer @@ -55,6 +56,10 @@ class Api::V1::Admin::DomainBlocksController < Api::BaseController private + def conflicts_with_existing_block?(domain_block, existing_domain_block) + existing_domain_block.present? && (existing_domain_block.domain == TagManager.instance.normalize_domain(domain_block.domain) || !domain_block.stricter_than?(existing_domain_block)) + end + def set_domain_blocks @domain_blocks = filtered_domain_blocks.order(id: :desc).to_a_paginated_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id)) end diff --git a/spec/requests/api/v1/admin/domain_blocks_spec.rb b/spec/requests/api/v1/admin/domain_blocks_spec.rb index a6cfdb48d..58641c20d 100644 --- a/spec/requests/api/v1/admin/domain_blocks_spec.rb +++ b/spec/requests/api/v1/admin/domain_blocks_spec.rb @@ -135,14 +135,10 @@ RSpec.describe 'Domain Blocks' do it_behaves_like 'forbidden for wrong role', '' it_behaves_like 'forbidden for wrong role', 'Moderator' - it 'returns http success' do + it 'creates a domain block with the expected domain name and severity', :aggregate_failures do subject expect(response).to have_http_status(200) - end - - it 'returns expected domain name and severity' do - subject body = body_as_json @@ -160,7 +156,44 @@ RSpec.describe 'Domain Blocks' do expect(DomainBlock.find_by(domain: 'foo.bar.com')).to be_present end - context 'when a stricter domain block already exists' do + context 'when a looser domain block already exists on a higher level domain' do + let(:params) { { domain: 'foo.bar.com', severity: :suspend } } + + before do + Fabricate(:domain_block, domain: 'bar.com', severity: :silence) + end + + it 'creates a domain block with the expected domain name and severity', :aggregate_failures do + subject + + body = body_as_json + + expect(response).to have_http_status(200) + expect(body).to match a_hash_including( + { + domain: 'foo.bar.com', + severity: 'suspend', + } + ) + + expect(DomainBlock.find_by(domain: 'foo.bar.com')).to be_present + end + end + + context 'when a domain block already exists on the same domain' do + before do + Fabricate(:domain_block, domain: 'foo.bar.com', severity: :silence) + end + + it 'returns existing domain block in error', :aggregate_failures do + subject + + expect(response).to have_http_status(422) + expect(body_as_json[:existing_domain_block][:domain]).to eq('foo.bar.com') + end + end + + context 'when a stricter domain block already exists on a higher level domain' do before do Fabricate(:domain_block, domain: 'bar.com', severity: :suspend) end From 67b2e62331f8459a3daa8ab06c8d18af4738e526 Mon Sep 17 00:00:00 2001 From: Emelia Smith Date: Tue, 30 Apr 2024 10:48:02 +0200 Subject: [PATCH 12/27] Fix missing destory audit logs for Domain Allows (#30125) --- app/controllers/admin/domain_allows_controller.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/controllers/admin/domain_allows_controller.rb b/app/controllers/admin/domain_allows_controller.rb index 31be1978b..b0f139e3a 100644 --- a/app/controllers/admin/domain_allows_controller.rb +++ b/app/controllers/admin/domain_allows_controller.rb @@ -25,6 +25,8 @@ class Admin::DomainAllowsController < Admin::BaseController def destroy authorize @domain_allow, :destroy? UnallowDomainService.new.call(@domain_allow) + log_action :destroy, @domain_allow + redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.destroyed_msg') end From 8cf78825a23184333a4d9ac775e81add5bd5f0da Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 15 May 2024 15:11:13 +0200 Subject: [PATCH 13/27] Fix off-by-one in `tootctl media` commands (#30306) --- lib/mastodon/cli/media.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/mastodon/cli/media.rb b/lib/mastodon/cli/media.rb index c90616177..5879c532e 100644 --- a/lib/mastodon/cli/media.rb +++ b/lib/mastodon/cli/media.rb @@ -134,7 +134,7 @@ module Mastodon::CLI model_name = path_segments.first.classify attachment_name = path_segments[1].singularize - record_id = path_segments[2..-2].join.to_i + record_id = path_segments[2...-2].join.to_i file_name = path_segments.last record = record_map.dig(model_name, record_id) attachment = record&.public_send(attachment_name) @@ -180,7 +180,7 @@ module Mastodon::CLI end model_name = path_segments.first.classify - record_id = path_segments[2..-2].join.to_i + record_id = path_segments[2...-2].join.to_i attachment_name = path_segments[1].singularize file_name = path_segments.last @@ -311,7 +311,7 @@ module Mastodon::CLI end model_name = path_segments.first.classify - record_id = path_segments[2..-2].join.to_i + record_id = path_segments[2...-2].join.to_i unless PRELOAD_MODEL_WHITELIST.include?(model_name) say("Cannot find corresponding model: #{model_name}", :red) @@ -361,7 +361,7 @@ module Mastodon::CLI next unless VALID_PATH_SEGMENTS_SIZE.include?(segments.size) model_name = segments.first.classify - record_id = segments[2..-2].join.to_i + record_id = segments[2...-2].join.to_i next unless PRELOAD_MODEL_WHITELIST.include?(model_name) From 8c72e80019ea01d50bd0a868441083688028615b Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 10 May 2024 15:59:50 +0200 Subject: [PATCH 14/27] Update dependency rack-cors to 2.0.2 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 94d2d0ba4..2deab967c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -537,7 +537,7 @@ GEM rack (2.2.8.1) rack-attack (6.7.0) rack (>= 1.0, < 4) - rack-cors (2.0.1) + rack-cors (2.0.2) rack (>= 2.0.0) rack-oauth2 (1.21.3) activesupport From 2865bfadafc1f4472e9012ff4f03329b5b19d321 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 10 May 2024 16:01:07 +0200 Subject: [PATCH 15/27] Update dependency json-jwt to 1.15.3.1 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 2deab967c..d08f79ccc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -368,7 +368,7 @@ GEM jmespath (1.6.2) json (2.6.3) json-canonicalization (1.0.0) - json-jwt (1.15.3) + json-jwt (1.15.3.1) activesupport (>= 4.2) aes_key_wrap bindata From 997b021b69e4f8fe8750e27c28bc3ffd5fe6cee8 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 10 May 2024 16:03:48 +0200 Subject: [PATCH 16/27] Update dependency rotp to 6.3.0 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index d08f79ccc..bf438bd34 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -606,7 +606,7 @@ GEM actionpack (>= 5.2) railties (>= 5.2) rexml (3.2.6) - rotp (6.2.2) + rotp (6.3.0) rouge (4.1.2) rpam2 (4.0.2) rqrcode (2.2.0) From 6fc07ff31f60a3c90c593c4c19ef9513f82a0124 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 10 May 2024 16:04:19 +0200 Subject: [PATCH 17/27] Update dependency fastimage to 2.3.1 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index bf438bd34..57783a522 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -288,7 +288,7 @@ GEM faraday_middleware (1.2.0) faraday (~> 1.0) fast_blank (1.0.1) - fastimage (2.2.7) + fastimage (2.3.1) ffi (1.15.5) ffi-compiler (1.0.1) ffi (>= 1.0.0) From a8dd32102fd5827c7a93df9ef6c2db21da43dde5 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 16 May 2024 10:11:21 +0200 Subject: [PATCH 18/27] Update dependency nokogiri to 1.16.5 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 57783a522..d9cb59c03 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -469,7 +469,7 @@ GEM net-protocol net-ssh (7.1.0) nio4r (2.7.0) - nokogiri (1.16.2) + nokogiri (1.16.5) mini_portile2 (~> 2.8.2) racc (~> 1.4) nsa (0.3.0) From 16213a678d391c18f283676648bbe235da3f6835 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 29 May 2024 11:09:54 +0200 Subject: [PATCH 19/27] Update dependency rexml to 3.2.8 --- Gemfile.lock | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index d9cb59c03..891566e1a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -605,7 +605,8 @@ GEM responders (3.1.0) actionpack (>= 5.2) railties (>= 5.2) - rexml (3.2.6) + rexml (3.2.8) + strscan (>= 3.0.9) rotp (6.3.0) rouge (4.1.2) rpam2 (4.0.2) @@ -731,6 +732,7 @@ GEM redlock (~> 1.0) strong_migrations (0.8.0) activerecord (>= 5.2) + strscan (3.0.9) swd (1.3.0) activesupport (>= 3) attr_required (>= 0.0.5) From b8edc95e8ad4ac07db311bc931cb09348b44820a Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 29 May 2024 10:15:06 +0200 Subject: [PATCH 20/27] Fix leaking Elasticsearch connections in Sidekiq processes (#30450) --- lib/mastodon/sidekiq_middleware.rb | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/lib/mastodon/sidekiq_middleware.rb b/lib/mastodon/sidekiq_middleware.rb index 3a747afb6..ac63d4d38 100644 --- a/lib/mastodon/sidekiq_middleware.rb +++ b/lib/mastodon/sidekiq_middleware.rb @@ -8,6 +8,7 @@ class Mastodon::SidekiqMiddleware rescue Mastodon::HostValidationError # Do not retry rescue => e + clean_up_elasticsearch_connections! limit_backtrace_and_raise(e) ensure clean_up_sockets! @@ -25,6 +26,32 @@ class Mastodon::SidekiqMiddleware clean_up_statsd_socket! end + # This is a hack to immediately free up unused Elasticsearch connections. + # + # Indeed, Chewy creates one `Elasticsearch::Client` instance per thread, + # and each such client manages its long-lasting connection to + # Elasticsearch. + # + # As far as I know, neither `chewy`, `elasticsearch-transport` or even + # `faraday` provide a reliable way to immediately close a connection, and + # rely on the underlying object to be garbage-collected instead. + # + # Furthermore, `sidekiq` creates a new thread each time a job throws an + # exception, meaning that each failure will create a new connection, and + # the old one will only be closed on full garbage collection. + def clean_up_elasticsearch_connections! + return unless Chewy.enabled? && Chewy.current[:chewy_client].present? + + Chewy.client.transport.connections.each do |connection| + # NOTE: This bit of code is tailored for the HTTPClient Faraday adapter + connection.connection.app.instance_variable_get(:@client)&.reset_all + end + + Chewy.current.delete(:chewy_client) + rescue + nil + end + def clean_up_redis_socket! RedisConfiguration.pool.checkin if Thread.current[:redis] Thread.current[:redis] = nil From f9c41ae43b24b3561f122dcb1e07d4cbd25b1806 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 23 May 2024 19:28:18 +0200 Subject: [PATCH 21/27] Normalize language code of incoming posts (#30403) --- app/lib/activitypub/parser/status_parser.rb | 11 +++- .../activitypub/parser/status_parser_spec.rb | 50 +++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 spec/lib/activitypub/parser/status_parser_spec.rb diff --git a/app/lib/activitypub/parser/status_parser.rb b/app/lib/activitypub/parser/status_parser.rb index 45f5fc5bf..6699dc21b 100644 --- a/app/lib/activitypub/parser/status_parser.rb +++ b/app/lib/activitypub/parser/status_parser.rb @@ -3,6 +3,8 @@ class ActivityPub::Parser::StatusParser include JsonLdHelper + NORMALIZED_LOCALE_NAMES = LanguagesHelper::SUPPORTED_LOCALES.keys.index_by(&:downcase).freeze + # @param [Hash] json # @param [Hash] magic_values # @option magic_values [String] :followers_collection @@ -86,6 +88,13 @@ class ActivityPub::Parser::StatusParser end def language + lang = raw_language_code + lang.presence && NORMALIZED_LOCALE_NAMES.fetch(lang.downcase.to_sym, lang) + end + + private + + def raw_language_code if content_language_map? @object['contentMap'].keys.first elsif name_language_map? @@ -95,8 +104,6 @@ class ActivityPub::Parser::StatusParser end end - private - def audience_to as_array(@object['to'] || @json['to']).map { |x| value_or_id(x) } end diff --git a/spec/lib/activitypub/parser/status_parser_spec.rb b/spec/lib/activitypub/parser/status_parser_spec.rb new file mode 100644 index 000000000..5d9f008db --- /dev/null +++ b/spec/lib/activitypub/parser/status_parser_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe ActivityPub::Parser::StatusParser do + subject { described_class.new(json) } + + let(:sender) { Fabricate(:account, followers_url: 'http://example.com/followers', domain: 'example.com', uri: 'https://example.com/actor') } + let(:follower) { Fabricate(:account, username: 'bob') } + + let(:json) do + { + '@context': 'https://www.w3.org/ns/activitystreams', + id: [ActivityPub::TagManager.instance.uri_for(sender), '#foo'].join, + type: 'Create', + actor: ActivityPub::TagManager.instance.uri_for(sender), + object: object_json, + }.with_indifferent_access + end + + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), 'post1'].join('/'), + type: 'Note', + to: [ + 'https://www.w3.org/ns/activitystreams#Public', + ActivityPub::TagManager.instance.uri_for(follower), + ], + content: '@bob lorem ipsum', + contentMap: { + EN: '@bob lorem ipsum', + }, + published: 1.hour.ago.utc.iso8601, + updated: 1.hour.ago.utc.iso8601, + tag: { + type: 'Mention', + href: ActivityPub::TagManager.instance.uri_for(follower), + }, + } + end + + it 'correctly parses status' do + expect(subject).to have_attributes( + text: '@bob lorem ipsum', + uri: [ActivityPub::TagManager.instance.uri_for(sender), 'post1'].join('/'), + reply: false, + language: :en + ) + end +end From 186f9161923f05cbc1999167caa9a2cf168749e9 Mon Sep 17 00:00:00 2001 From: Emelia Smith Date: Wed, 29 May 2024 16:00:05 +0200 Subject: [PATCH 22/27] Fix: remove broken OAuth Application vacuuming & throttle OAuth Application registrations (#30316) Co-authored-by: Claire --- app/lib/vacuum/applications_vacuum.rb | 10 ---- app/workers/scheduler/vacuum_scheduler.rb | 5 -- config/initializers/rack_attack.rb | 4 ++ spec/config/initializers/rack_attack_spec.rb | 18 ++++++++ spec/lib/vacuum/applications_vacuum_spec.rb | 48 -------------------- 5 files changed, 22 insertions(+), 63 deletions(-) delete mode 100644 app/lib/vacuum/applications_vacuum.rb delete mode 100644 spec/lib/vacuum/applications_vacuum_spec.rb diff --git a/app/lib/vacuum/applications_vacuum.rb b/app/lib/vacuum/applications_vacuum.rb deleted file mode 100644 index ba88655f1..000000000 --- a/app/lib/vacuum/applications_vacuum.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -class Vacuum::ApplicationsVacuum - def perform - Doorkeeper::Application.where(owner_id: nil) - .where.missing(:created_users, :access_tokens, :access_grants) - .where(created_at: ...1.day.ago) - .in_batches.delete_all - end -end diff --git a/app/workers/scheduler/vacuum_scheduler.rb b/app/workers/scheduler/vacuum_scheduler.rb index 1c9a2aabe..c22d6f5f8 100644 --- a/app/workers/scheduler/vacuum_scheduler.rb +++ b/app/workers/scheduler/vacuum_scheduler.rb @@ -22,7 +22,6 @@ class Scheduler::VacuumScheduler preview_cards_vacuum, backups_vacuum, access_tokens_vacuum, - applications_vacuum, feeds_vacuum, imports_vacuum, ] @@ -56,10 +55,6 @@ class Scheduler::VacuumScheduler Vacuum::ImportsVacuum.new end - def applications_vacuum - Vacuum::ApplicationsVacuum.new - end - def content_retention_policy ContentRetentionPolicy.current end diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index d0af0fe94..482f4b95a 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -105,6 +105,10 @@ class Rack::Attack req.authenticated_user_id if (req.post? && req.path.match?(API_DELETE_REBLOG_REGEX)) || (req.delete? && req.path.match?(API_DELETE_STATUS_REGEX)) end + throttle('throttle_oauth_application_registrations/ip', limit: 5, period: 10.minutes) do |req| + req.throttleable_remote_ip if req.post? && req.path == '/api/v1/apps' + end + throttle('throttle_sign_up_attempts/ip', limit: 25, period: 5.minutes) do |req| req.throttleable_remote_ip if req.post? && req.path_matches?('/auth') end diff --git a/spec/config/initializers/rack_attack_spec.rb b/spec/config/initializers/rack_attack_spec.rb index 7cd4ac76b..ddbd74705 100644 --- a/spec/config/initializers/rack_attack_spec.rb +++ b/spec/config/initializers/rack_attack_spec.rb @@ -103,4 +103,22 @@ describe Rack::Attack, type: :request do it_behaves_like 'throttled endpoint' end end + + describe 'throttle excessive oauth application registration requests by IP address' do + let(:throttle) { 'throttle_oauth_application_registrations/ip' } + let(:limit) { 5 } + let(:period) { 10.minutes } + let(:path) { '/api/v1/apps' } + let(:params) do + { + client_name: 'Throttle Test', + redirect_uris: 'urn:ietf:wg:oauth:2.0:oob', + scopes: 'read', + } + end + + let(:request) { -> { post path, params: params, headers: { 'REMOTE_ADDR' => remote_ip } } } + + it_behaves_like 'throttled endpoint' + end end diff --git a/spec/lib/vacuum/applications_vacuum_spec.rb b/spec/lib/vacuum/applications_vacuum_spec.rb deleted file mode 100644 index 57a222aaf..000000000 --- a/spec/lib/vacuum/applications_vacuum_spec.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -RSpec.describe Vacuum::ApplicationsVacuum do - subject { described_class.new } - - describe '#perform' do - let!(:app_with_token) { Fabricate(:application, created_at: 1.month.ago) } - let!(:app_with_grant) { Fabricate(:application, created_at: 1.month.ago) } - let!(:app_with_signup) { Fabricate(:application, created_at: 1.month.ago) } - let!(:app_with_owner) { Fabricate(:application, created_at: 1.month.ago, owner: Fabricate(:user)) } - let!(:unused_app) { Fabricate(:application, created_at: 1.month.ago) } - let!(:recent_app) { Fabricate(:application, created_at: 1.hour.ago) } - - let!(:active_access_token) { Fabricate(:access_token, application: app_with_token) } - let!(:active_access_grant) { Fabricate(:access_grant, application: app_with_grant) } - let!(:user) { Fabricate(:user, created_by_application: app_with_signup) } - - before do - subject.perform - end - - it 'does not delete applications with valid access tokens' do - expect { app_with_token.reload }.to_not raise_error - end - - it 'does not delete applications with valid access grants' do - expect { app_with_grant.reload }.to_not raise_error - end - - it 'does not delete applications that were used to create users' do - expect { app_with_signup.reload }.to_not raise_error - end - - it 'does not delete owned applications' do - expect { app_with_owner.reload }.to_not raise_error - end - - it 'does not delete applications registered less than a day ago' do - expect { recent_app.reload }.to_not raise_error - end - - it 'deletes unused applications' do - expect { unused_app.reload }.to raise_error ActiveRecord::RecordNotFound - end - end -end From 943792c187392dc0a0f0b4e34dae5ddbc5c169b3 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 30 May 2024 14:03:13 +0200 Subject: [PATCH 23/27] Merge pull request from GHSA-5fq7-3p3j-9vrf --- app/services/notify_service.rb | 17 +++++++++-------- spec/services/notify_service_spec.rb | 8 ++++---- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb index 125883b15..759d6e393 100644 --- a/app/services/notify_service.rb +++ b/app/services/notify_service.rb @@ -71,16 +71,17 @@ class NotifyService < BaseService LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id WHERE s.id = :id UNION ALL - SELECT s.id, s.in_reply_to_id, m.id, st.path || s.id, st.depth + 1 - FROM ancestors st - JOIN statuses s ON s.id = st.in_reply_to_id - LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id - WHERE st.mention_id IS NULL AND NOT s.id = ANY(path) AND st.depth < :depth_limit + SELECT s.id, s.in_reply_to_id, m.id, ancestors.path || s.id, ancestors.depth + 1 + FROM ancestors + JOIN statuses s ON s.id = ancestors.in_reply_to_id + /* early exit if we already have a mention matching our requirements */ + LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id AND s.account_id = :recipient_id + WHERE ancestors.mention_id IS NULL AND NOT s.id = ANY(path) AND ancestors.depth < :depth_limit ) SELECT COUNT(*) - FROM ancestors st - JOIN statuses s ON s.id = st.id - WHERE st.mention_id IS NOT NULL AND s.visibility = 3 + FROM ancestors + JOIN statuses s ON s.id = ancestors.id + WHERE ancestors.mention_id IS NOT NULL AND s.account_id = :recipient_id AND s.visibility = 3 SQL end diff --git a/spec/services/notify_service_spec.rb b/spec/services/notify_service_spec.rb index 8fcb58658..c2664e79c 100644 --- a/spec/services/notify_service_spec.rb +++ b/spec/services/notify_service_spec.rb @@ -76,10 +76,10 @@ RSpec.describe NotifyService, type: :service do end context 'when the message chain is initiated by recipient, but without a mention to the sender, even if the sender sends multiple messages in a row' do - let(:reply_to) { Fabricate(:status, account: recipient) } - let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) } - let(:dummy_reply) { Fabricate(:status, account: sender, visibility: :direct, thread: reply_to) } - let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: dummy_reply)) } + let(:public_status) { Fabricate(:status, account: recipient) } + let(:intermediate_reply) { Fabricate(:status, account: sender, thread: public_status, visibility: :direct) } + let!(:intermediate_mention) { Fabricate(:mention, account: sender, status: intermediate_reply) } + let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: intermediate_reply)) } it 'does not notify' do expect { subject }.to_not change(Notification, :count) From 7920aa59e85bf21a4f6e2e9d485cd82dac89554c Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 30 May 2024 14:14:04 +0200 Subject: [PATCH 24/27] Merge pull request from GHSA-q3rg-xx5v-4mxh --- config/initializers/rack_attack.rb | 10 +++++++- spec/config/initializers/rack_attack_spec.rb | 24 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index 482f4b95a..426b14b25 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -30,13 +30,17 @@ class Rack::Attack end def authenticated_user_id - authenticated_token&.resource_owner_id + authenticated_token&.resource_owner_id || warden_user_id end def authenticated_token_id authenticated_token&.id end + def warden_user_id + @env['warden']&.user&.id + end + def unauthenticated? !authenticated_user_id end @@ -141,6 +145,10 @@ class Rack::Attack req.session[:attempt_user_id] || req.params.dig('user', 'email').presence if req.post? && req.path_matches?('/auth/sign_in') end + throttle('throttle_password_change/account', limit: 10, period: 10.minutes) do |req| + req.authenticated_user_id if req.put? || (req.patch? && req.path_matches?('/auth')) + end + self.throttled_responder = lambda do |request| now = Time.now.utc match_data = request.env['rack.attack.match_data'] diff --git a/spec/config/initializers/rack_attack_spec.rb b/spec/config/initializers/rack_attack_spec.rb index ddbd74705..78c4bf03a 100644 --- a/spec/config/initializers/rack_attack_spec.rb +++ b/spec/config/initializers/rack_attack_spec.rb @@ -121,4 +121,28 @@ describe Rack::Attack, type: :request do it_behaves_like 'throttled endpoint' end + + describe 'throttle excessive password change requests by account' do + let(:user) { Fabricate(:user, email: 'user@host.example') } + let(:limit) { 10 } + let(:period) { 10.minutes } + let(:request) { -> { put path, headers: { 'REMOTE_ADDR' => remote_ip } } } + let(:path) { '/auth' } + + before do + sign_in user, scope: :user + + # Unfortunately, devise's `sign_in` helper causes the `session` to be + # loaded in the next request regardless of whether it's actually accessed + # by the client code. + # + # So, we make an extra query to clear issue a session cookie instead. + # + # A less resource-intensive way to deal with that would be to generate the + # session cookie manually, but this seems pretty involved. + get '/' + end + + it_behaves_like 'throttled endpoint' + end end From 8ab0ca7d64b9f7ec9d658d9a90eae1120ccef117 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 30 May 2024 14:24:29 +0200 Subject: [PATCH 25/27] Merge pull request from GHSA-c2r5-cfqr-c553 * Add hardening monkey-patch to prevent IP spoofing on misconfigured installations * Remove rack-attack safelist --- config/application.rb | 1 + config/initializers/rack_attack.rb | 4 -- lib/action_dispatch/remote_ip_extensions.rb | 72 +++++++++++++++++++++ 3 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 lib/action_dispatch/remote_ip_extensions.rb diff --git a/config/application.rb b/config/application.rb index 2a62c37e8..c2a63c869 100644 --- a/config/application.rb +++ b/config/application.rb @@ -48,6 +48,7 @@ require_relative '../lib/chewy/strategy/bypass_with_warning' require_relative '../lib/webpacker/manifest_extensions' require_relative '../lib/webpacker/helper_extensions' require_relative '../lib/rails/engine_extensions' +require_relative '../lib/action_dispatch/remote_ip_extensions' require_relative '../lib/active_record/database_tasks_extensions' require_relative '../lib/active_record/batches' require_relative '../lib/simple_navigation/item_extensions' diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index 426b14b25..962f02994 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -62,10 +62,6 @@ class Rack::Attack end end - Rack::Attack.safelist('allow from localhost') do |req| - req.remote_ip == '127.0.0.1' || req.remote_ip == '::1' - end - Rack::Attack.blocklist('deny from blocklist') do |req| IpBlock.blocked?(req.remote_ip) end diff --git a/lib/action_dispatch/remote_ip_extensions.rb b/lib/action_dispatch/remote_ip_extensions.rb new file mode 100644 index 000000000..e5c48bf3c --- /dev/null +++ b/lib/action_dispatch/remote_ip_extensions.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +# Mastodon is not made to be directly accessed without a reverse proxy. +# This monkey-patch prevents remote IP address spoofing when being accessed +# directly. +# +# See PR: https://github.com/rails/rails/pull/51610 + +# In addition to the PR above, it also raises an error if a request with +# `X-Forwarded-For` or `Client-Ip` comes directly from a client without +# going through a trusted proxy. + +# rubocop:disable all -- This is a mostly vendored file + +module ActionDispatch + class RemoteIp + module GetIpExtensions + def calculate_ip + # Set by the Rack web server, this is a single value. + remote_addr = ips_from(@req.remote_addr).last + + # Could be a CSV list and/or repeated headers that were concatenated. + client_ips = ips_from(@req.client_ip).reverse! + forwarded_ips = ips_from(@req.x_forwarded_for).reverse! + + # `Client-Ip` and `X-Forwarded-For` should not, generally, both be set. If they + # are both set, it means that either: + # + # 1) This request passed through two proxies with incompatible IP header + # conventions. + # + # 2) The client passed one of `Client-Ip` or `X-Forwarded-For` + # (whichever the proxy servers weren't using) themselves. + # + # Either way, there is no way for us to determine which header is the right one + # after the fact. Since we have no idea, if we are concerned about IP spoofing + # we need to give up and explode. (If you're not concerned about IP spoofing you + # can turn the `ip_spoofing_check` option off.) + should_check_ip = @check_ip && client_ips.last && forwarded_ips.last + if should_check_ip && !forwarded_ips.include?(client_ips.last) + # We don't know which came from the proxy, and which from the user + raise IpSpoofAttackError, "IP spoofing attack?! " \ + "HTTP_CLIENT_IP=#{@req.client_ip.inspect} " \ + "HTTP_X_FORWARDED_FOR=#{@req.x_forwarded_for.inspect}" + end + + # NOTE: Mastodon addition to make sure we don't get requests from a non-trusted client + if @check_ip && (forwarded_ips.last || client_ips.last) && !@proxies.any? { |proxy| proxy === remote_addr } + raise IpSpoofAttackError, "IP spoofing attack?! client #{remote_addr} is not a trusted proxy " \ + "HTTP_CLIENT_IP=#{@req.client_ip.inspect} " \ + "HTTP_X_FORWARDED_FOR=#{@req.x_forwarded_for.inspect}" + end + + # We assume these things about the IP headers: + # + # - X-Forwarded-For will be a list of IPs, one per proxy, or blank + # - Client-Ip is propagated from the outermost proxy, or is blank + # - REMOTE_ADDR will be the IP that made the request to Rack + ips = forwarded_ips + client_ips + ips.compact! + + # If every single IP option is in the trusted list, return the IP that's + # furthest away + filter_proxies([remote_addr] + ips).first || ips.last || remote_addr + end + end + end +end + +ActionDispatch::RemoteIp::GetIp.prepend(ActionDispatch::RemoteIp::GetIpExtensions) + +# rubocop:enable all From 9740c7eaea07629a4d6154a3ed26995110737a0c Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 30 May 2024 14:56:18 +0200 Subject: [PATCH 26/27] Fix rate-limiting incorrectly triggering a session cookie on most endpoints (#30483) --- config/initializers/rack_attack.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index 962f02994..6d8284e2b 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -30,7 +30,7 @@ class Rack::Attack end def authenticated_user_id - authenticated_token&.resource_owner_id || warden_user_id + authenticated_token&.resource_owner_id end def authenticated_token_id @@ -142,7 +142,7 @@ class Rack::Attack end throttle('throttle_password_change/account', limit: 10, period: 10.minutes) do |req| - req.authenticated_user_id if req.put? || (req.patch? && req.path_matches?('/auth')) + req.warden_user_id if req.put? || (req.patch? && req.path_matches?('/auth')) end self.throttled_responder = lambda do |request| From c93aacafdea188cac791b62a32e3117a7dc3e9cc Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 30 May 2024 15:34:50 +0200 Subject: [PATCH 27/27] Bump version to v4.2.9 (#30470) --- CHANGELOG.md | 35 +++++++++++++++++++++++++++++++++++ docker-compose.yml | 6 +++--- lib/mastodon/version.rb | 2 +- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 636b3c161..8e579e148 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,41 @@ All notable changes to this project will be documented in this file. +## [4.2.9] - 2024-05-30 + +### Security + +- Update dependencies +- Fix private mention filtering ([GHSA-5fq7-3p3j-9vrf](https://github.com/mastodon/mastodon/security/advisories/GHSA-5fq7-3p3j-9vrf)) +- Fix password change endpoint not being rate-limited ([GHSA-q3rg-xx5v-4mxh](https://github.com/mastodon/mastodon/security/advisories/GHSA-q3rg-xx5v-4mxh)) +- Add hardening around rate-limit bypass ([GHSA-c2r5-cfqr-c553](https://github.com/mastodon/mastodon/security/advisories/GHSA-c2r5-cfqr-c553)) + +### Added + +- Add rate-limit on OAuth application registration ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30316)) +- Add fallback redirection when getting a webfinger query `WEB_DOMAIN@WEB_DOMAIN` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28592)) +- Add `digest` attribute to `Admin::DomainBlock` entity in REST API ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/29092)) + +### Removed + +- Remove superfluous application-level caching in some controllers ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29862)) +- Remove aggressive OAuth application vacuuming ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30316)) + +### Fixed + +- Fix leaking Elasticsearch connections in Sidekiq processes ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30450)) +- Fix language of remote posts not being recognized when using unusual casing ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30403)) +- Fix off-by-one in `tootctl media` commands ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30306)) +- Fix removal of allowed domains (in `LIMITED_FEDERATION_MODE`) not being recorded in the audit log ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30125)) +- Fix not being able to block a subdomain of an already-blocked domain through the API ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30119)) +- Fix `Idempotency-Key` being ignored when scheduling a post ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30084)) +- Fix crash when supplying the `FFMPEG_BINARY` environment variable ([timothyjrogers](https://github.com/mastodon/mastodon/pull/30022)) +- Fix improper email address validation ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29838)) +- Fix results/query in `api/v1/featured_tags/suggestions` ([mjankowski](https://github.com/mastodon/mastodon/pull/29597)) +- Fix unblocking internationalized domain names under certain conditions ([tribela](https://github.com/mastodon/mastodon/pull/29530)) +- Fix admin account created by `mastodon:setup` not being auto-approved ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29379)) +- Fix reference to non-existent var in CLI maintenance command ([mjankowski](https://github.com/mastodon/mastodon/pull/28363)) + ## [4.2.8] - 2024-02-23 ### Added diff --git a/docker-compose.yml b/docker-compose.yml index 004f5dbb0..10a57e31e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,7 +56,7 @@ services: web: build: . - image: ghcr.io/mastodon/mastodon:v4.2.8 + image: ghcr.io/mastodon/mastodon:v4.2.9 restart: always env_file: .env.production command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" @@ -77,7 +77,7 @@ services: streaming: build: . - image: ghcr.io/mastodon/mastodon:v4.2.8 + image: ghcr.io/mastodon/mastodon:v4.2.9 restart: always env_file: .env.production command: node ./streaming @@ -95,7 +95,7 @@ services: sidekiq: build: . - image: ghcr.io/mastodon/mastodon:v4.2.8 + image: ghcr.io/mastodon/mastodon:v4.2.9 restart: always env_file: .env.production command: bundle exec sidekiq diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index b28349dce..f9088382f 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 8 + 9 end def default_prerelease