From 76323a0cb23b27e4b2dd0e08dbce731d98507188 Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 17 Feb 2024 01:06:01 +1030 Subject: [PATCH 01/17] Update .env.production.sample --- .env.production.sample | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.env.production.sample b/.env.production.sample index 0bf01bdc361..cb5bdcd5d46 100644 --- a/.env.production.sample +++ b/.env.production.sample @@ -75,3 +75,11 @@ S3_ALIAS_HOST=files.example.com # ----------------------- IP_RETENTION_PERIOD=31556952 SESSION_RETENTION_PERIOD=31556952 + +# Antispam: IP Block/Limit lists (Limited IP require manual approval). +# ----------------------- +# URLS in this list are refreshed every 24 hours. IP Addresses are removed after for 24 hours +# Comma delimited to add more URLs. The format should contain only IP addresses (subnet optional) +# ----------------------- +# SCHEDULED_IPLIMIT_URLS="https://check.torproject.org/torbulkexitlist" +# SCHEDULED_IPBLOCK_URLS="https://check.torproject.org/torbulkexitlist" From eff406b4c8268a8540e6b617d2263574abffbd99 Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 17 Feb 2024 01:07:10 +1030 Subject: [PATCH 02/17] Update .env.production.sample --- .env.production.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.production.sample b/.env.production.sample index cb5bdcd5d46..5068dfbee55 100644 --- a/.env.production.sample +++ b/.env.production.sample @@ -82,4 +82,4 @@ SESSION_RETENTION_PERIOD=31556952 # Comma delimited to add more URLs. The format should contain only IP addresses (subnet optional) # ----------------------- # SCHEDULED_IPLIMIT_URLS="https://check.torproject.org/torbulkexitlist" -# SCHEDULED_IPBLOCK_URLS="https://check.torproject.org/torbulkexitlist" +# SCHEDULED_IPBLOCK_URLS="" From afa1c150dae327d1f93957d044b14f208482668e Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 17 Feb 2024 01:13:50 +1030 Subject: [PATCH 03/17] Create ip_spamlist_urls_scheduler.rb --- .../scheduler/ip_spamlist_urls_scheduler.rb | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 app/workers/scheduler/ip_spamlist_urls_scheduler.rb diff --git a/app/workers/scheduler/ip_spamlist_urls_scheduler.rb b/app/workers/scheduler/ip_spamlist_urls_scheduler.rb new file mode 100644 index 00000000000..9fedeb467a3 --- /dev/null +++ b/app/workers/scheduler/ip_spamlist_urls_scheduler.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +class Scheduler::IPBlocklistURLScheduler + include Sidekiq::Worker + + sidekiq_options retry: 0 + + def perform + if ENV['SCHEDULED_IPBLOCK_URLS'].present? + @block_url = ENV['SCHEDULED_IPBLOCK_URLS'] + @blockips = [] + grab_ip_blocks_addresses! + add_ip_blocks_addresses! + end + + if ENV['SCHEDULED_IPLIMIT_URLS'].present? + @limit_url = ENV['SCHEDULED_IPBLIMIT_URLS'] + @limitips = [] + grab_ip_limit_addresses! + add_ip_limit_addresses! + end + end + + def grab_ip_blocks_addresses! + @block_url.split(',').each do |url| + Request.new(:get, url).perform do |res| + @blockips.insert = res.body + end + end + end + + def grab_ip_limit_addresses! + @limit_url.split(',').each do |url| + Request.new(:get, url).perform do |res| + @limitips.insert = res.body + end + end + end + + def add_ip_blocks_addresses! + @blockips.each do |ip| + ip_block = IpBlock.find_by(ip: ip) + + if ip_block.present? + ip_block.update(expires_in: 24.hours.to_i) + next + end + + IpBlock.create( + ip: ip, + severity: :sign_up_block, + comment: 'Scheduled IPBlock', + expires_in: 24.hours.to_i + ) + end + end + + def add_ip_limit_addresses! + @limitips.each do |ip| + ip_limit = Iplimit.find_by(ip: ip) + + if ip_limit.present? + ip_limit.update(expires_in: 24.hours.to_i) + next + end + + Iplimit.create( + ip: ip, + severity: :sign_up_requires_approval, + comment: 'Scheduled IPLimit', + expires_in: 24.hours.to_i + ) + end + end + +end From 97c396edaf44a72f9bfe92a8a1008fce105cbc1f Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 17 Feb 2024 01:14:39 +1030 Subject: [PATCH 04/17] Update sidekiq.yml --- config/sidekiq.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 3f9cbd9a7a5..9faf08bde42 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -63,3 +63,7 @@ interval: 30 minutes class: Scheduler::SoftwareUpdateCheckScheduler queue: scheduler + ip_spamlist_URL_scheduler: + every: '24h' + class: Scheduler::IPSpamlistURLScheduler + queue: scheduler From 860d4c1da8e3bf2bd6c690b1dc7ac8a6e9627a5f Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 17 Feb 2024 01:15:49 +1030 Subject: [PATCH 05/17] Update ip_spamlist_urls_scheduler.rb --- app/workers/scheduler/ip_spamlist_urls_scheduler.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/scheduler/ip_spamlist_urls_scheduler.rb b/app/workers/scheduler/ip_spamlist_urls_scheduler.rb index 9fedeb467a3..b0c6ee725e8 100644 --- a/app/workers/scheduler/ip_spamlist_urls_scheduler.rb +++ b/app/workers/scheduler/ip_spamlist_urls_scheduler.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class Scheduler::IPBlocklistURLScheduler +class Scheduler::IPSpamlistURLScheduler include Sidekiq::Worker sidekiq_options retry: 0 From 47b25d80afb458c7c6c70e0b1f4fb1b464a169db Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 17 Feb 2024 01:18:57 +1030 Subject: [PATCH 06/17] Update ip_spamlist_urls_scheduler.rb --- app/workers/scheduler/ip_spamlist_urls_scheduler.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/workers/scheduler/ip_spamlist_urls_scheduler.rb b/app/workers/scheduler/ip_spamlist_urls_scheduler.rb index b0c6ee725e8..5fca6cd11f5 100644 --- a/app/workers/scheduler/ip_spamlist_urls_scheduler.rb +++ b/app/workers/scheduler/ip_spamlist_urls_scheduler.rb @@ -57,14 +57,14 @@ class Scheduler::IPSpamlistURLScheduler def add_ip_limit_addresses! @limitips.each do |ip| - ip_limit = Iplimit.find_by(ip: ip) + ip_limit = IpBlock.find_by(ip: ip) if ip_limit.present? ip_limit.update(expires_in: 24.hours.to_i) next end - Iplimit.create( + IpBlock.create( ip: ip, severity: :sign_up_requires_approval, comment: 'Scheduled IPLimit', From 1b0a477a8f1d42ec962643b1dd8504a286d55995 Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 17 Feb 2024 01:23:35 +1030 Subject: [PATCH 07/17] Update .env.production.sample --- .env.production.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.production.sample b/.env.production.sample index 5068dfbee55..e56cdfdc4d0 100644 --- a/.env.production.sample +++ b/.env.production.sample @@ -78,7 +78,7 @@ SESSION_RETENTION_PERIOD=31556952 # Antispam: IP Block/Limit lists (Limited IP require manual approval). # ----------------------- -# URLS in this list are refreshed every 24 hours. IP Addresses are removed after for 24 hours +# URLS in this list are refreshed every 24 hours. Blocked/limited IP Addresses are removed after 24 hours. # Comma delimited to add more URLs. The format should contain only IP addresses (subnet optional) # ----------------------- # SCHEDULED_IPLIMIT_URLS="https://check.torproject.org/torbulkexitlist" From dba64bac83544995e0432cbc5cfba6106e817077 Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 17 Feb 2024 01:24:29 +1030 Subject: [PATCH 08/17] Update ip_spamlist_urls_scheduler.rb --- app/workers/scheduler/ip_spamlist_urls_scheduler.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/app/workers/scheduler/ip_spamlist_urls_scheduler.rb b/app/workers/scheduler/ip_spamlist_urls_scheduler.rb index 5fca6cd11f5..ed92b171846 100644 --- a/app/workers/scheduler/ip_spamlist_urls_scheduler.rb +++ b/app/workers/scheduler/ip_spamlist_urls_scheduler.rb @@ -72,5 +72,4 @@ class Scheduler::IPSpamlistURLScheduler ) end end - end From 05ece28852d0837927e0df3d9664249b4268a066 Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 17 Feb 2024 01:24:59 +1030 Subject: [PATCH 09/17] Update .env.production.sample --- .env.production.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.production.sample b/.env.production.sample index e56cdfdc4d0..cedf878992d 100644 --- a/.env.production.sample +++ b/.env.production.sample @@ -78,7 +78,7 @@ SESSION_RETENTION_PERIOD=31556952 # Antispam: IP Block/Limit lists (Limited IP require manual approval). # ----------------------- -# URLS in this list are refreshed every 24 hours. Blocked/limited IP Addresses are removed after 24 hours. +# URLS in this list are refreshed every 24 hours. Blocked/limited IP Addresses expire after 24 hours. # Comma delimited to add more URLs. The format should contain only IP addresses (subnet optional) # ----------------------- # SCHEDULED_IPLIMIT_URLS="https://check.torproject.org/torbulkexitlist" From 56fe3a5d1adb7902fcf5494828316765cd8338c0 Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 17 Feb 2024 01:29:08 +1030 Subject: [PATCH 10/17] Update ip_spamlist_urls_scheduler.rb From 99535b4f4652b42241533b47987d633c4f6541fe Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 17 Feb 2024 01:41:06 +1030 Subject: [PATCH 11/17] Update ip_spamlist_urls_scheduler.rb --- .../scheduler/ip_spamlist_urls_scheduler.rb | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/app/workers/scheduler/ip_spamlist_urls_scheduler.rb b/app/workers/scheduler/ip_spamlist_urls_scheduler.rb index ed92b171846..336f77ba6c5 100644 --- a/app/workers/scheduler/ip_spamlist_urls_scheduler.rb +++ b/app/workers/scheduler/ip_spamlist_urls_scheduler.rb @@ -6,19 +6,15 @@ class Scheduler::IPSpamlistURLScheduler sidekiq_options retry: 0 def perform - if ENV['SCHEDULED_IPBLOCK_URLS'].present? - @block_url = ENV['SCHEDULED_IPBLOCK_URLS'] - @blockips = [] - grab_ip_blocks_addresses! - add_ip_blocks_addresses! - end - - if ENV['SCHEDULED_IPLIMIT_URLS'].present? - @limit_url = ENV['SCHEDULED_IPBLIMIT_URLS'] - @limitips = [] - grab_ip_limit_addresses! - add_ip_limit_addresses! - end + @block_url = ENV['SCHEDULED_IPBLOCK_URLS'] if ENV['SCHEDULED_IPBLOCK_URLS'].present? + @blockips = [] + grab_ip_blocks_addresses! if ENV['SCHEDULED_IPBLOCK_URLS'].present? + add_ip_blocks_addresses! if ENV['SCHEDULED_IPBLOCK_URLS'].present? + + @limit_url = ENV['SCHEDULED_IPBLIMIT_URLS'] if ENV['SCHEDULED_IPLIMIT_URLS'].present? + @limitips = [] + grab_ip_limit_addresses! if ENV['SCHEDULED_IPLIMIT_URLS'].present? + add_ip_limit_addresses! if ENV['SCHEDULED_IPLIMIT_URLS'].present? end def grab_ip_blocks_addresses! From 396208aded02f76bd560d52710f7c7cbce282f96 Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 17 Feb 2024 01:44:07 +1030 Subject: [PATCH 12/17] Update ip_spamlist_urls_scheduler.rb --- app/workers/scheduler/ip_spamlist_urls_scheduler.rb | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/app/workers/scheduler/ip_spamlist_urls_scheduler.rb b/app/workers/scheduler/ip_spamlist_urls_scheduler.rb index 336f77ba6c5..e18e0302ea7 100644 --- a/app/workers/scheduler/ip_spamlist_urls_scheduler.rb +++ b/app/workers/scheduler/ip_spamlist_urls_scheduler.rb @@ -6,19 +6,15 @@ class Scheduler::IPSpamlistURLScheduler sidekiq_options retry: 0 def perform - @block_url = ENV['SCHEDULED_IPBLOCK_URLS'] if ENV['SCHEDULED_IPBLOCK_URLS'].present? - @blockips = [] grab_ip_blocks_addresses! if ENV['SCHEDULED_IPBLOCK_URLS'].present? add_ip_blocks_addresses! if ENV['SCHEDULED_IPBLOCK_URLS'].present? - - @limit_url = ENV['SCHEDULED_IPBLIMIT_URLS'] if ENV['SCHEDULED_IPLIMIT_URLS'].present? - @limitips = [] grab_ip_limit_addresses! if ENV['SCHEDULED_IPLIMIT_URLS'].present? add_ip_limit_addresses! if ENV['SCHEDULED_IPLIMIT_URLS'].present? end def grab_ip_blocks_addresses! - @block_url.split(',').each do |url| + @blockips = [] + ENV['SCHEDULED_IPBLOCK_URLS'].split(',').each do |url| Request.new(:get, url).perform do |res| @blockips.insert = res.body end @@ -26,7 +22,8 @@ class Scheduler::IPSpamlistURLScheduler end def grab_ip_limit_addresses! - @limit_url.split(',').each do |url| + @limitips = [] + ENV['SCHEDULED_IPBLIMIT_URLS'].split(',').each do |url| Request.new(:get, url).perform do |res| @limitips.insert = res.body end From 2f23cc799b008841fb407420356e4532343be0c4 Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 17 Feb 2024 01:44:30 +1030 Subject: [PATCH 13/17] Update ip_spamlist_urls_scheduler.rb --- app/workers/scheduler/ip_spamlist_urls_scheduler.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/scheduler/ip_spamlist_urls_scheduler.rb b/app/workers/scheduler/ip_spamlist_urls_scheduler.rb index e18e0302ea7..df96bfd7651 100644 --- a/app/workers/scheduler/ip_spamlist_urls_scheduler.rb +++ b/app/workers/scheduler/ip_spamlist_urls_scheduler.rb @@ -64,5 +64,5 @@ class Scheduler::IPSpamlistURLScheduler expires_in: 24.hours.to_i ) end - end + end end From 5fdbb708339d8d9433e0024a523eddda35fadf94 Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 17 Feb 2024 01:54:23 +1030 Subject: [PATCH 14/17] Update ip_spamlist_urls_scheduler.rb --- app/workers/scheduler/ip_spamlist_urls_scheduler.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/scheduler/ip_spamlist_urls_scheduler.rb b/app/workers/scheduler/ip_spamlist_urls_scheduler.rb index df96bfd7651..5574d1a56b1 100644 --- a/app/workers/scheduler/ip_spamlist_urls_scheduler.rb +++ b/app/workers/scheduler/ip_spamlist_urls_scheduler.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class Scheduler::IPSpamlistURLScheduler +class Scheduler::IpSpamlistUrlsScheduler include Sidekiq::Worker sidekiq_options retry: 0 From 44877ebd847e32c5d46a00bf01ca9a698e43e11b Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 17 Feb 2024 01:54:36 +1030 Subject: [PATCH 15/17] Update sidekiq.yml --- config/sidekiq.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 9faf08bde42..ddb6c4b8248 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -65,5 +65,5 @@ queue: scheduler ip_spamlist_URL_scheduler: every: '24h' - class: Scheduler::IPSpamlistURLScheduler + class: Scheduler::IpSpamlistUrlsScheduler queue: scheduler From 6c50ab4bbe6d19d17e90669b072de3a474e97660 Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 17 Feb 2024 01:57:01 +1030 Subject: [PATCH 16/17] Update sidekiq.yml --- config/sidekiq.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/sidekiq.yml b/config/sidekiq.yml index ddb6c4b8248..feff3c86130 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -63,7 +63,7 @@ interval: 30 minutes class: Scheduler::SoftwareUpdateCheckScheduler queue: scheduler - ip_spamlist_URL_scheduler: + ip_spamlist_url_scheduler: every: '24h' class: Scheduler::IpSpamlistUrlsScheduler queue: scheduler From 4e4bdaf335bc7838fa59a8985dd7245c5a722fe3 Mon Sep 17 00:00:00 2001 From: Shlee Date: Wed, 27 Mar 2024 12:55:56 +1030 Subject: [PATCH 17/17] Update sidekiq.yml --- config/sidekiq.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 771a504bc94..51b497008ce 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -70,4 +70,4 @@ ip_spamlist_url_scheduler: every: '24h' class: Scheduler::IpSpamlistUrlsScheduler - queue: scheduler \ No newline at end of file + queue: scheduler