Merge remote-tracking branch 'localupstream/428' into deploy

CHANGELOG.md

@@ -2,6 +2,26 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.2.8] - 2024-02-23
+
+### Added
+
+- Add hourly task to automatically require approval for new registrations in the absence of moderators ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29318), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/29355))
+  In order to prevent future abandoned Mastodon servers from being used for spam, harassment and other malicious activity, Mastodon will now automatically switch new user registrations to require moderator approval whenever they are left open and no activity (including non-moderation actions from apps) from any logged-in user with permission to access moderation reports has been detected in a full week.
+  When this happens, users with the permission to change server settings will receive an email notification.
+  This feature is disabled when `EMAIL_DOMAIN_ALLOWLIST` is used, and can also be disabled with `DISABLE_AUTOMATIC_SWITCHING_TO_APPROVED_REGISTRATIONS=true`.
+
+### Changed
+
+- Change registrations to be closed by default on new installations ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29280))
+  If you are running a server and never changed your registrations mode from the default, updating will automatically close your registrations.
+  Simply re-enable them through the administration interface or using `tootctl settings registrations open` if you want to enable them again.
+
+### Fixed
+
+- Fix processing of remote ActivityPub actors making use of `Link` objects as `Image` `url` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29335))
+- Fix link verifications when page size exceeds 1MB ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29358))
+
 ## [4.2.7] - 2024-02-16
 
 ### Fixed

Gemfile.lock

@@ -28,47 +28,47 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (7.0.8)
+    actioncable (7.0.8.1)
-      actionpack (= 7.0.8)
+      actionpack (= 7.0.8.1)
-      activesupport (= 7.0.8)
+      activesupport (= 7.0.8.1)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (7.0.8)
+    actionmailbox (7.0.8.1)
-      actionpack (= 7.0.8)
+      actionpack (= 7.0.8.1)
-      activejob (= 7.0.8)
+      activejob (= 7.0.8.1)
-      activerecord (= 7.0.8)
+      activerecord (= 7.0.8.1)
-      activestorage (= 7.0.8)
+      activestorage (= 7.0.8.1)
-      activesupport (= 7.0.8)
+      activesupport (= 7.0.8.1)
       mail (>= 2.7.1)
       net-imap
       net-pop
       net-smtp
-    actionmailer (7.0.8)
+    actionmailer (7.0.8.1)
-      actionpack (= 7.0.8)
+      actionpack (= 7.0.8.1)
-      actionview (= 7.0.8)
+      actionview (= 7.0.8.1)
-      activejob (= 7.0.8)
+      activejob (= 7.0.8.1)
-      activesupport (= 7.0.8)
+      activesupport (= 7.0.8.1)
       mail (~> 2.5, >= 2.5.4)
       net-imap
       net-pop
       net-smtp
       rails-dom-testing (~> 2.0)
-    actionpack (7.0.8)
+    actionpack (7.0.8.1)
-      actionview (= 7.0.8)
+      actionview (= 7.0.8.1)
-      activesupport (= 7.0.8)
+      activesupport (= 7.0.8.1)
       rack (~> 2.0, >= 2.2.4)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (7.0.8)
+    actiontext (7.0.8.1)
-      actionpack (= 7.0.8)
+      actionpack (= 7.0.8.1)
-      activerecord (= 7.0.8)
+      activerecord (= 7.0.8.1)
-      activestorage (= 7.0.8)
+      activestorage (= 7.0.8.1)
-      activesupport (= 7.0.8)
+      activesupport (= 7.0.8.1)
       globalid (>= 0.6.0)
       nokogiri (>= 1.8.5)
-    actionview (7.0.8)
+    actionview (7.0.8.1)
-      activesupport (= 7.0.8)
+      activesupport (= 7.0.8.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
@@ -78,22 +78,22 @@ GEM
       activemodel (>= 4.1, < 7.1)
       case_transform (>= 0.2)
       jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
-    activejob (7.0.8)
+    activejob (7.0.8.1)
-      activesupport (= 7.0.8)
+      activesupport (= 7.0.8.1)
       globalid (>= 0.3.6)
-    activemodel (7.0.8)
+    activemodel (7.0.8.1)
-      activesupport (= 7.0.8)
+      activesupport (= 7.0.8.1)
-    activerecord (7.0.8)
+    activerecord (7.0.8.1)
-      activemodel (= 7.0.8)
+      activemodel (= 7.0.8.1)
-      activesupport (= 7.0.8)
+      activesupport (= 7.0.8.1)
-    activestorage (7.0.8)
+    activestorage (7.0.8.1)
-      actionpack (= 7.0.8)
+      actionpack (= 7.0.8.1)
-      activejob (= 7.0.8)
+      activejob (= 7.0.8.1)
-      activerecord (= 7.0.8)
+      activerecord (= 7.0.8.1)
-      activesupport (= 7.0.8)
+      activesupport (= 7.0.8.1)
       marcel (~> 1.0)
       mini_mime (>= 1.1.0)
-    activesupport (7.0.8)
+    activesupport (7.0.8.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -215,7 +215,7 @@ GEM
       activerecord (>= 5.a)
       database_cleaner-core (~> 2.0.0)
     database_cleaner-core (2.0.1)
-    date (3.3.3)
+    date (3.3.4)
     debug_inspector (1.1.0)
     devise (4.9.2)
       bcrypt (~> 3.0)
@@ -424,7 +424,7 @@ GEM
       activesupport (>= 4)
       railties (>= 4)
       request_store (~> 1.0)
-    loofah (2.21.3)
+    loofah (2.21.4)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     mail (2.8.1)
@@ -461,11 +461,11 @@ GEM
     net-ldap (0.18.0)
     net-pop (0.1.2)
       net-protocol
-    net-protocol (0.2.1)
+    net-protocol (0.2.2)
       timeout
     net-scp (4.0.0)
       net-ssh (>= 2.6.5, < 8.0.0)
-    net-smtp (0.3.3)
+    net-smtp (0.3.4)
       net-protocol
     net-ssh (7.1.0)
     nio4r (2.7.0)
@@ -534,7 +534,7 @@ GEM
       activesupport (>= 3.0.0)
     raabro (1.4.0)
     racc (1.7.3)
-    rack (2.2.8)
+    rack (2.2.8.1)
     rack-attack (6.7.0)
       rack (>= 1.0, < 4)
     rack-cors (2.0.1)
@@ -551,20 +551,20 @@ GEM
       rack
     rack-test (2.1.0)
       rack (>= 1.3)
-    rails (7.0.8)
+    rails (7.0.8.1)
-      actioncable (= 7.0.8)
+      actioncable (= 7.0.8.1)
-      actionmailbox (= 7.0.8)
+      actionmailbox (= 7.0.8.1)
-      actionmailer (= 7.0.8)
+      actionmailer (= 7.0.8.1)
-      actionpack (= 7.0.8)
+      actionpack (= 7.0.8.1)
-      actiontext (= 7.0.8)
+      actiontext (= 7.0.8.1)
-      actionview (= 7.0.8)
+      actionview (= 7.0.8.1)
-      activejob (= 7.0.8)
+      activejob (= 7.0.8.1)
-      activemodel (= 7.0.8)
+      activemodel (= 7.0.8.1)
-      activerecord (= 7.0.8)
+      activerecord (= 7.0.8.1)
-      activestorage (= 7.0.8)
+      activestorage (= 7.0.8.1)
-      activesupport (= 7.0.8)
+      activesupport (= 7.0.8.1)
       bundler (>= 1.15.0)
-      railties (= 7.0.8)
+      railties (= 7.0.8.1)
     rails-controller-testing (1.0.5)
       actionpack (>= 5.0.1.rc1)
       actionview (>= 5.0.1.rc1)
@@ -579,9 +579,9 @@ GEM
     rails-i18n (7.0.7)
       i18n (>= 0.7, < 2)
       railties (>= 6.0.0, < 8)
-    railties (7.0.8)
+    railties (7.0.8.1)
-      actionpack (= 7.0.8)
+      actionpack (= 7.0.8.1)
-      activesupport (= 7.0.8)
+      activesupport (= 7.0.8.1)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
@@ -744,7 +744,7 @@ GEM
     test-prof (1.2.3)
     thor (1.3.0)
     tilt (2.2.0)
-    timeout (0.4.0)
+    timeout (0.4.1)
     tpm-key_attestation (0.12.0)
       bindata (~> 2.4)
       openssl (> 2.0)
@@ -808,7 +808,7 @@ GEM
     xorcist (1.1.3)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.6.11)
+    zeitwerk (2.6.13)
 
 PLATFORMS
   ruby

SECURITY.md

@@ -13,8 +13,8 @@ A "vulnerability in Mastodon" is a vulnerability in the code distributed through
 
 ## Supported Versions
 
-| Version | Supported        |
+| Version | Supported |
-| ------- | ---------------- |
+| ------- | --------- |
-| 4.2.x   | Yes              |
+| 4.2.x   | Yes       |
-| 4.1.x   | Yes              |
+| 4.1.x   | Yes       |
-| < 4.1   | No               |
+| < 4.1   | No        |

app/javascript/packs/admin.jsx

@@ -145,6 +145,10 @@ delegate(document, '#form_admin_settings_enable_bootstrap_timeline_accounts', 'c
 const onChangeRegistrationMode = (target) => {
   const enabled = target.value === 'approved';
 
+  [].forEach.call(document.querySelectorAll('.form_admin_settings_registrations_mode .warning-hint'), (warning_hint) => {
+    warning_hint.style.display = target.value === 'open' ? 'inline' : 'none';
+  });
+
   [].forEach.call(document.querySelectorAll('#form_admin_settings_require_invite_text'), (input) => {
     input.disabled = !enabled;
     if (enabled) {

app/mailers/admin_mailer.rb

@@ -61,6 +61,12 @@ class AdminMailer < ApplicationMailer
     end
   end
 
+  def auto_close_registrations
+    locale_for_account(@me) do
+      mail subject: default_i18n_subject(instance: @instance)
+    end
+  end
+
   private
 
   def process_params

app/services/activitypub/process_account_service.rb

@@ -201,10 +201,15 @@ class ActivityPub::ProcessAccountService < BaseService
     value = first_of_value(@json[key])
 
     return if value.nil?
-    return value['url'] if value.is_a?(Hash)
 
-    image = fetch_resource_without_id_validation(value)
+    if value.is_a?(String)
-    image['url'] if image
+      value = fetch_resource_without_id_validation(value)
+      return if value.nil?
+    end
+
+    value = first_of_value(value['url']) if value.is_a?(Hash) && value['type'] == 'Image'
+    value = value['href'] if value.is_a?(Hash)
+    value if value.is_a?(String)
   end
 
   def public_key

app/services/verify_link_service.rb

@@ -19,7 +19,7 @@ class VerifyLinkService < BaseService
 
   def perform_request!
     @body = Request.new(:get, @url).add_headers('Accept' => 'text/html').perform do |res|
-      res.code == 200 ? res.body_with_limit : nil
+      res.code == 200 ? res.truncated_body : nil
     end
   end
 

app/views/admin/settings/registrations/show.html.haml

@@ -10,9 +10,11 @@
 
   %p.lead= t('admin.settings.registrations.preamble')
 
+  .flash-message= t('admin.settings.registrations.moderation_recommandation')
+
   .fields-row
     .fields-row__column.fields-row__column-6.fields-group
-      = f.input :registrations_mode, collection: %w(open approved none), wrapper: :with_label, include_blank: false, label_method: ->(mode) { I18n.t("admin.settings.registrations_mode.modes.#{mode}") }
+      = f.input :registrations_mode, collection: %w(open approved none), wrapper: :with_label, include_blank: false, label_method: ->(mode) { I18n.t("admin.settings.registrations_mode.modes.#{mode}") }, warning_hint: I18n.t('admin.settings.registrations_mode.warning_hint')
 
     .fields-row__column.fields-row__column-6.fields-group
       = f.input :require_invite_text, as: :boolean, wrapper: :with_label, disabled: !approved_registrations?

app/views/admin_mailer/auto_close_registrations.text.erb

@@ -0,0 +1,3 @@
+<%= raw t('admin_mailer.auto_close_registrations.body', instance: @instance) %>
+
+<%= raw t('application_mailer.view')%> <%= admin_settings_registrations_url %>

app/workers/scheduler/auto_close_registrations_scheduler.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+class Scheduler::AutoCloseRegistrationsScheduler
+  include Sidekiq::Worker
+  include Redisable
+
+  sidekiq_options retry: 0
+
+  # Automatically switch away from open registrations if no
+  # moderator had any activity in that period of time
+  OPEN_REGISTRATIONS_MODERATOR_THRESHOLD = 1.week + UserTrackingConcern::SIGN_IN_UPDATE_FREQUENCY
+
+  def perform
+    return if Rails.configuration.x.email_domains_whitelist.present? || ENV['DISABLE_AUTOMATIC_SWITCHING_TO_APPROVED_REGISTRATIONS'] == 'true'
+    return unless Setting.registrations_mode == 'open'
+
+    switch_to_approval_mode! unless active_moderators?
+  end
+
+  private
+
+  def active_moderators?
+    User.those_who_can(:manage_reports).exists?(current_sign_in_at: OPEN_REGISTRATIONS_MODERATOR_THRESHOLD.ago...)
+  end
+
+  def switch_to_approval_mode!
+    Setting.registrations_mode = 'approved'
+
+    User.those_who_can(:manage_settings).includes(:account).find_each do |user|
+      AdminMailer.with(recipient: user.account).auto_close_registrations.deliver_later
+    end
+  end
+end

config/locales/en.yml

@@ -764,6 +764,7 @@ en:
         disabled: To no one
         users: To logged-in local users
       registrations:
+        moderation_recommandation: Please make sure you have an adequate and reactive moderation team before you open registrations to everyone!
         preamble: Control who can create an account on your server.
         title: Registrations
       registrations_mode:
@@ -771,6 +772,7 @@ en:
           approved: Approval required for sign up
           none: Nobody can sign up
           open: Anyone can sign up
+        warning_hint: We recommend using “Approval required for sign up” unless you are confident your moderation team can handle spam and malicious registrations in a timely fashion.
       security:
         authorized_fetch: Require authentication from federated servers
         authorized_fetch_hint: Requiring authentication from federated servers enables stricter enforcement of both user-level and server-level blocks. However, this comes at the cost of a performance penalty, reduces the reach of your replies, and may introduce compatibility issues with some federated services. In addition, this will not prevent dedicated actors from fetching your public posts and accounts.
@@ -963,6 +965,9 @@ en:
       title: Webhooks
       webhook: Webhook
   admin_mailer:
+    auto_close_registrations:
+      body: Due to a lack of recent moderator activity, registrations on %{instance} have been automatically switched to requiring manual review, to prevent %{instance} from being used as a platform for potential bad actors. You can switch it back to open registrations at any time.
+      subject: Registrations for %{instance} have been automatically switched to requiring approval
     new_appeal:
       actions:
         delete_statuses: to delete their posts

config/settings.yml

@@ -9,7 +9,7 @@ defaults: &defaults
   site_terms: ''
   site_contact_username: ''
   site_contact_email: ''
-  registrations_mode: 'open'
+  registrations_mode: 'none'
   profile_directory: true
   closed_registrations_message: ''
   timeline_preview: true

config/sidekiq.yml

@@ -62,3 +62,7 @@
       interval: 30 minutes
       class: Scheduler::SoftwareUpdateCheckScheduler
       queue: scheduler
+    auto_close_registrations_scheduler:
+      interval: 1 hour
+      class: Scheduler::AutoCloseRegistrationsScheduler
+      queue: scheduler

lib/mastodon/version.rb

@@ -13,7 +13,7 @@ module Mastodon
     end
 
     def patch
-      7
+      8
     end
 
     def default_prerelease

spec/models/setting_spec.rb

@@ -142,22 +142,12 @@ RSpec.describe Setting do
     context 'when records includes nothing' do
       let(:records) { [] }
 
-      context 'when default_value is not a Hash' do
+      it 'includes Setting with value of default_value' do
-        it 'includes Setting with value of default_value' do
+        setting = described_class.all_as_records[key]
-          setting = described_class.all_as_records[key]
 
-          expect(setting).to be_a described_class
+        expect(setting).to be_a described_class
-          expect(setting).to have_attributes(var: key)
+        expect(setting).to have_attributes(var: key)
-          expect(setting).to have_attributes(value: 'default_value')
+        expect(setting).to have_attributes(value: default_value)
-        end
-      end
-
-      context 'when default_value is a Hash' do
-        let(:default_value) { { 'foo' => 'fuga' } }
-
-        it 'returns {}' do
-          expect(described_class.all_as_records).to eq({})
-        end
       end
     end
   end

spec/rails_helper.rb

@@ -152,6 +152,13 @@ RSpec.configure do |config|
     self.use_transactional_tests = false
 
     DatabaseCleaner.cleaning do
+      # NOTE: we switched registrations mode to closed by default, but the specs
+      # very heavily rely on having it enabled by default, as it relies on users
+      # being approved by default except in select cases where explicitly testing
+      # other registration modes
+      # Also needs to be set per-example here because of the database cleaner.
+      Setting.registrations_mode = 'open'
+
       example.run
     end
 

spec/services/activitypub/process_account_service_spec.rb

@@ -5,7 +5,7 @@ require 'rails_helper'
 RSpec.describe ActivityPub::ProcessAccountService, type: :service do
   subject { described_class.new }
 
-  context 'with property values' do
+  context 'with property values, an avatar, and a profile header' do
     let(:payload) do
       {
         id: 'https://foo.test',
@@ -16,19 +16,50 @@ RSpec.describe ActivityPub::ProcessAccountService, type: :service do
           { type: 'PropertyValue', name: 'Occupation', value: 'Unit test' },
           { type: 'PropertyValue', name: 'non-string', value: %w(foo bar) },
         ],
+        image: {
+          type: 'Image',
+          mediaType: 'image/png',
+          url: 'https://foo.test/image.png',
+        },
+        icon: {
+          type: 'Image',
+          url: [
+            {
+              mediaType: 'image/png',
+              href: 'https://foo.test/icon.png',
+            },
+          ],
+        },
       }.with_indifferent_access
     end
 
-    it 'parses out of attachment' do
+    before do
+      stub_request(:get, 'https://foo.test/image.png').to_return(request_fixture('avatar.txt'))
+      stub_request(:get, 'https://foo.test/icon.png').to_return(request_fixture('avatar.txt'))
+    end
+
+    it 'parses property values, avatar and profile header as expected' do
       account = subject.call('alice', 'example.com', payload)
-      expect(account.fields).to be_a Array
+
-      expect(account.fields.size).to eq 2
+      expect(account.fields)
-      expect(account.fields[0]).to be_a Account::Field
+        .to be_an(Array)
-      expect(account.fields[0].name).to eq 'Pronouns'
+        .and have_attributes(size: 2)
-      expect(account.fields[0].value).to eq 'They/them'
+      expect(account.fields.first)
-      expect(account.fields[1]).to be_a Account::Field
+        .to be_an(Account::Field)
-      expect(account.fields[1].name).to eq 'Occupation'
+        .and have_attributes(
-      expect(account.fields[1].value).to eq 'Unit test'
+          name: eq('Pronouns'),
+          value: eq('They/them')
+        )
+      expect(account.fields.last)
+        .to be_an(Account::Field)
+        .and have_attributes(
+          name: eq('Occupation'),
+          value: eq('Unit test')
+        )
+      expect(account).to have_attributes(
+        avatar_remote_url: 'https://foo.test/icon.png',
+        header_remote_url: 'https://foo.test/image.png'
+      )
     end
   end
 

spec/services/verify_link_service_spec.rb

@@ -76,6 +76,20 @@ RSpec.describe VerifyLinkService, type: :service do
     end
 
     context 'when a document is truncated but the link back is valid' do
+      let(:html) do
+        "
+          <!doctype html>
+          <body>
+            <a rel=\"me\" href=\"#{ActivityPub::TagManager.instance.url_for(account)}\">
+        "
+      end
+
+      it 'marks the field as verified' do
+        expect(field.verified?).to be true
+      end
+    end
+
+    context 'when a link tag might be truncated' do
       let(:html) do
         "
           <!doctype html>
@@ -89,19 +103,6 @@ RSpec.describe VerifyLinkService, type: :service do
       end
     end
 
-    context 'when a link back might be truncated' do
-      let(:html) do
-        "
-          <!doctype html>
-          <body>
-            <a rel=\"me\" href=\"#{ActivityPub::TagManager.instance.url_for(account)}"
-      end
-
-      it 'does not mark the field as verified' do
-        expect(field.verified?).to be false
-      end
-    end
-
     context 'when a link does not contain a link back' do
       let(:html) { '' }
 

spec/spec_helper.rb

@@ -31,6 +31,12 @@ RSpec.configure do |config|
   config.before :suite do
     Rails.application.load_seed
     Chewy.strategy(:bypass)
+
+    # NOTE: we switched registrations mode to closed by default, but the specs
+    # very heavily rely on having it enabled by default, as it relies on users
+    # being approved by default except in select cases where explicitly testing
+    # other registration modes
+    Setting.registrations_mode = 'open'
   end
 
   config.after :suite do

spec/workers/scheduler/auto_close_registrations_scheduler_spec.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe Scheduler::AutoCloseRegistrationsScheduler do
+  subject { described_class.new }
+
+  describe '#perform' do
+    let(:moderator_activity_date) { Time.now.utc }
+
+    before do
+      Fabricate(:user, role: UserRole.find_by(name: 'Owner'), current_sign_in_at: 10.years.ago)
+      Fabricate(:user, role: UserRole.find_by(name: 'Moderator'), current_sign_in_at: moderator_activity_date)
+    end
+
+    context 'when registrations are open' do
+      before do
+        Setting.registrations_mode = 'open'
+      end
+
+      context 'when a moderator has logged in recently' do
+        let(:moderator_activity_date) { Time.now.utc }
+
+        it 'does not change registrations mode' do
+          expect { subject.perform }.to_not change(Setting, :registrations_mode)
+        end
+      end
+
+      context 'when a moderator has not recently signed in' do
+        let(:moderator_activity_date) { 1.year.ago }
+
+        it 'changes registrations mode from open to approved' do
+          expect { subject.perform }.to change(Setting, :registrations_mode).from('open').to('approved')
+        end
+      end
+    end
+
+    context 'when registrations are closed' do
+      before do
+        Setting.registrations_mode = 'none'
+      end
+
+      context 'when a moderator has logged in recently' do
+        let(:moderator_activity_date) { Time.now.utc }
+
+        it 'does not change registrations mode' do
+          expect { subject.perform }.to_not change(Setting, :registrations_mode)
+        end
+      end
+
+      context 'when a moderator has not recently signed in' do
+        let(:moderator_activity_date) { 1.year.ago }
+
+        it 'does not change registrations mode' do
+          expect { subject.perform }.to_not change(Setting, :registrations_mode)
+        end
+      end
+    end
+  end
+end