From 4d1b67f664e463f28ff45b8e125998ffcd2de50b Mon Sep 17 00:00:00 2001 From: Renaud Chaput Date: Fri, 28 Jul 2023 23:09:49 +0200 Subject: [PATCH] Add end-to-end (system) tests (#25461) --- .github/workflows/test-ruby.yml | 97 +++++++++++++++++++++++++ Gemfile | 4 + Gemfile.lock | 11 +++ config/application.rb | 2 +- config/webpack/tests.js | 2 +- lib/tasks/spec.rake | 11 +++ spec/rails_helper.rb | 48 +++++++++++- spec/spec_helper.rb | 77 ++++++++++++++++++++ spec/support/stories/profile_stories.rb | 6 ++ spec/system/new_statuses_spec.rb | 45 ++++++++++++ 10 files changed, 298 insertions(+), 5 deletions(-) create mode 100644 lib/tasks/spec.rake create mode 100644 spec/system/new_statuses_spec.rb diff --git a/.github/workflows/test-ruby.yml b/.github/workflows/test-ruby.yml index ee9eefd4586..ff135867f93 100644 --- a/.github/workflows/test-ruby.yml +++ b/.github/workflows/test-ruby.yml @@ -153,3 +153,100 @@ jobs: run: './bin/rails db:create db:schema:load db:seed' - run: bundle exec rake rspec_chunked + + test-e2e: + name: End to End testing + runs-on: ubuntu-latest + + needs: + - build + + services: + postgres: + image: postgres:14-alpine + env: + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + redis: + image: redis:7-alpine + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 + + env: + DB_HOST: localhost + DB_USER: postgres + DB_PASS: postgres + DISABLE_SIMPLECOV: true + RAILS_ENV: test + BUNDLE_WITH: test + + strategy: + fail-fast: false + matrix: + ruby-version: + - '3.0' + - '3.1' + - '.ruby-version' + + steps: + - uses: actions/checkout@v3 + + - uses: actions/download-artifact@v3 + with: + path: './public' + name: ${{ github.sha }} + + - name: Update package index + run: sudo apt-get update + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + cache: yarn + node-version-file: '.nvmrc' + + - name: Install native Ruby dependencies + run: sudo apt-get install -y libicu-dev libidn11-dev + + - name: Install additional system dependencies + run: sudo apt-get install -y ffmpeg imagemagick + + - name: Set up bundler cache + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version}} + bundler-cache: true + + - run: yarn --frozen-lockfile + + - name: Load database schema + run: './bin/rails db:create db:schema:load db:seed' + + - run: bundle exec rake spec:system + + - name: Archive logs + uses: actions/upload-artifact@v3 + if: failure() + with: + name: e2e-logs-${{ matrix.ruby-version }} + path: log/ + + - name: Archive test screenshots + uses: actions/upload-artifact@v3 + if: failure() + with: + name: e2e-screenshots + path: tmp/screenshots/ diff --git a/Gemfile b/Gemfile index fcd10c5f9b6..ff9a9cdb161 100644 --- a/Gemfile +++ b/Gemfile @@ -113,6 +113,10 @@ group :test do # Browser integration testing gem 'capybara', '~> 3.39' + gem 'selenium-webdriver' + + # Used to reset the database between system tests + gem 'database_cleaner-active_record' # Used to mock environment variables gem 'climate_control', '~> 0.2' diff --git a/Gemfile.lock b/Gemfile.lock index 5b1c62a692e..fda288c6f0d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -199,6 +199,10 @@ GEM crass (1.0.6) css_parser (1.14.0) addressable + database_cleaner-active_record (2.1.0) + activerecord (>= 5.a) + database_cleaner-core (~> 2.0.0) + database_cleaner-core (2.0.1) date (3.3.3) debug_inspector (1.1.0) devise (4.9.2) @@ -656,6 +660,10 @@ GEM scenic (1.7.0) activerecord (>= 4.0.0) railties (>= 4.0.0) + selenium-webdriver (4.9.1) + rexml (~> 3.2, >= 3.2.5) + rubyzip (>= 1.2.2, < 3.0) + websocket (~> 1.0) semantic_range (3.0.0) sidekiq (6.5.9) connection_pool (>= 2.2.5, < 3) @@ -768,6 +776,7 @@ GEM rack-proxy (>= 0.6.1) railties (>= 5.2) semantic_range (>= 2.3.0) + websocket (1.2.9) websocket-driver (0.7.5) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) @@ -804,6 +813,7 @@ DEPENDENCIES color_diff (~> 0.1) concurrent-ruby connection_pool + database_cleaner-active_record devise (~> 4.9) devise-two-factor (~> 4.1) devise_pam_authenticatable2 (~> 9.2) @@ -885,6 +895,7 @@ DEPENDENCIES rubyzip (~> 2.3) sanitize (~> 6.0) scenic (~> 1.7) + selenium-webdriver sidekiq (~> 6.5) sidekiq-bulk (~> 0.2.0) sidekiq-scheduler (~> 5.0) diff --git a/config/application.rb b/config/application.rb index 6f21efa8db4..436d7b3307a 100644 --- a/config/application.rb +++ b/config/application.rb @@ -199,7 +199,7 @@ module Mastodon # We use our own middleware for this config.public_file_server.enabled = false - config.middleware.use PublicFileServerMiddleware if Rails.env.development? || ENV['RAILS_SERVE_STATIC_FILES'] == 'true' + config.middleware.use PublicFileServerMiddleware if Rails.env.development? || Rails.env.test? || ENV['RAILS_SERVE_STATIC_FILES'] == 'true' config.middleware.use Rack::Attack config.middleware.use Mastodon::RackMiddleware diff --git a/config/webpack/tests.js b/config/webpack/tests.js index 1f7bdea9daa..e6a8f1c2a95 100644 --- a/config/webpack/tests.js +++ b/config/webpack/tests.js @@ -5,5 +5,5 @@ const { merge } = require('webpack-merge'); const sharedConfig = require('./shared'); module.exports = merge(sharedConfig, { - mode: 'development', + mode: 'production', }); diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake new file mode 100644 index 00000000000..8f2cbeea358 --- /dev/null +++ b/lib/tasks/spec.rake @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +if Rake::Task.task_defined?('spec:system') + namespace :spec do + task :enable_system_specs do # rubocop:disable Rails/RakeEnvironment + ENV['RUN_SYSTEM_SPECS'] = 'true' + end + end + + Rake::Task['spec:system'].enhance ['spec:enable_system_specs'] +end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 2645f74e40b..0f1073630d9 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -1,6 +1,14 @@ # frozen_string_literal: true ENV['RAILS_ENV'] ||= 'test' + +# This needs to be defined before Rails is initialized +RUN_SYSTEM_SPECS = ENV.fetch('RUN_SYSTEM_SPECS', false) + +if RUN_SYSTEM_SPECS + STREAMING_PORT = ENV.fetch('TEST_STREAMING_PORT', '4020') + ENV['STREAMING_API_BASE_URL'] = "http://localhost:#{STREAMING_PORT}" +end require File.expand_path('../config/environment', __dir__) abort('The Rails environment is running in production mode!') if Rails.env.production? @@ -15,10 +23,14 @@ require 'chewy/rspec' Dir[Rails.root.join('spec', 'support', '**', '*.rb')].each { |f| require f } ActiveRecord::Migration.maintain_test_schema! -WebMock.disable_net_connect!(allow: Chewy.settings[:host]) +WebMock.disable_net_connect!(allow: Chewy.settings[:host], allow_localhost: RUN_SYSTEM_SPECS) Sidekiq::Testing.inline! Sidekiq.logger = nil +# System tests config +DatabaseCleaner.strategy = [:deletion] +streaming_server_manager = StreamingServerManager.new + Devise::Test::ControllerHelpers.module_eval do alias_method :original_sign_in, :sign_in @@ -56,6 +68,8 @@ module SignedRequestHelpers end RSpec.configure do |config| + # This is set before running spec:system, see lib/tasks/tests.rake + config.filter_run_excluding type: :system unless RUN_SYSTEM_SPECS config.fixture_path = Rails.root.join('spec', 'fixtures') config.use_transactional_fixtures = true config.order = 'random' @@ -83,8 +97,7 @@ RSpec.configure do |config| end config.before :each, type: :feature do - https = ENV['LOCAL_HTTPS'] == 'true' - Capybara.app_host = "http#{https ? 's' : ''}://#{ENV.fetch('LOCAL_DOMAIN')}" + Capybara.current_driver = :rack_test end config.before :each, type: :controller do @@ -95,6 +108,35 @@ RSpec.configure do |config| stub_jsonld_contexts! end + config.before :suite do + if RUN_SYSTEM_SPECS + Webpacker.compile + streaming_server_manager.start(port: STREAMING_PORT) + end + end + + config.after :suite do + streaming_server_manager.stop + end + + config.around :each, type: :system do |example| + # driven_by :selenium, using: :chrome, screen_size: [1600, 1200] + driven_by :selenium, using: :headless_chrome, screen_size: [1600, 1200] + + # The streaming server needs access to the database + # but with use_transactional_tests every transaction + # is rolled-back, so the streaming server never sees the data + # So we disable this feature for system tests, and use DatabaseCleaner to clean + # the database tables between each test + self.use_transactional_tests = false + + DatabaseCleaner.cleaning do + example.run + end + + self.use_transactional_tests = true + end + config.before(:each) do |example| unless example.metadata[:paperclip_processing] allow_any_instance_of(Paperclip::Attachment).to receive(:post_process).and_return(true) # rubocop:disable RSpec/AnyInstance diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 7b3af0f90bc..dcbcad48e6f 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -52,3 +52,80 @@ def expect_push_bulk_to_match(klass, matcher) 'args' => matcher, })) end + +class StreamingServerManager + @running_thread = nil + + def initialize + at_exit { stop } + end + + def start(port: 4020) + return if @running_thread + + queue = Queue.new + + @queue = queue + + @running_thread = Thread.new do + Open3.popen2e( + { + 'REDIS_NAMESPACE' => ENV.fetch('REDIS_NAMESPACE'), + 'DB_NAME' => "#{ENV.fetch('DB_NAME', 'mastodon')}_test#{ENV.fetch('TEST_ENV_NUMBER', '')}", + 'RAILS_ENV' => ENV.fetch('RAILS_ENV', 'test'), + 'NODE_ENV' => ENV.fetch('STREAMING_NODE_ENV', 'development'), + 'PORT' => port.to_s, + }, + 'node index.js', # must not call yarn here, otherwise it will fail because yarn does not send signals to its child process + chdir: Rails.root.join('streaming') + ) do |_stdin, stdout_err, process_thread| + status = :starting + + # Spawn a thread to listen on streaming server output + output_thread = Thread.new do + stdout_err.each_line do |line| + Rails.logger.info "Streaming server: #{line}" + + if status == :starting && line.match('Streaming API now listening on') + status = :started + @queue.enq 'started' + end + end + end + + # And another thread to listen on commands from the main thread + loop do + msg = queue.pop + + case msg + when 'stop' + # we need to properly stop the reading thread + output_thread.kill + + # Then stop the node process + Process.kill('KILL', process_thread.pid) + + # And we stop ourselves + @running_thread.kill + end + end + end + end + + # wait for 10 seconds for the streaming server to start + Timeout.timeout(10) do + loop do + break if @queue.pop == 'started' + end + end + end + + def stop + return unless @running_thread + + @queue.enq 'stop' + + # Wait for the thread to end + @running_thread.join + end +end diff --git a/spec/support/stories/profile_stories.rb b/spec/support/stories/profile_stories.rb index de7ae17e633..2b345ddef10 100644 --- a/spec/support/stories/profile_stories.rb +++ b/spec/support/stories/profile_stories.rb @@ -9,6 +9,8 @@ module ProfileStories email: email, password: password, confirmed_at: confirmed_at, account: Fabricate(:account, username: 'bob') ) + + Web::Setting.where(user: bob).first_or_initialize(user: bob).update!(data: { introductionVersion: 201812160442020 }) if finished_onboarding # rubocop:disable Style/NumericLiterals end def as_a_logged_in_user @@ -42,4 +44,8 @@ module ProfileStories def password @password ||= 'password' end + + def finished_onboarding + @finished_onboarding || false + end end diff --git a/spec/system/new_statuses_spec.rb b/spec/system/new_statuses_spec.rb new file mode 100644 index 00000000000..6faed6c808c --- /dev/null +++ b/spec/system/new_statuses_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe 'NewStatuses' do + include ProfileStories + + subject { page } + + let(:email) { 'test@example.com' } + let(:password) { 'password' } + let(:confirmed_at) { Time.zone.now } + let(:finished_onboarding) { true } + + before do + as_a_logged_in_user + visit root_path + end + + it 'can be posted' do + expect(subject).to have_css('div.app-holder') + + status_text = 'This is a new status!' + + within('.compose-form') do + fill_in "What's on your mind?", with: status_text + click_on 'Publish!' + end + + expect(subject).to have_selector('.status__content__text', text: status_text) + end + + it 'can be posted again' do + expect(subject).to have_css('div.app-holder') + + status_text = 'This is a second status!' + + within('.compose-form') do + fill_in "What's on your mind?", with: status_text + click_on 'Publish!' + end + + expect(subject).to have_selector('.status__content__text', text: status_text) + end +end