erik/mastodon
1
0
Fork 0
mirror of https://github.com/mastodon/mastodon.git synced 2024-08-20 21:08:15 -07:00
Code Issues Projects Releases Wiki Activity

Add size limit for all PreviewCard URLs (#30973)

Browse source
This commit is contained in:
David Roetzel 2024-07-09 15:11:34 +02:00 committed by GitHub
parent ef2e48e6da
commit 967505ee9b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 35 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
app/models/preview_card.rb
View file

@ -46,6 +46,11 @@ class PreviewCard < ApplicationRecord
y_comp: 4, y_comp: 4,
}.freeze }.freeze
# URL size limit to safely store in PosgreSQL's unique indexes
# Technically this is a byte-size limit but we use it as a
# character limit to work with length validation
URL_CHARACTER_LIMIT = 2692
self.inheritance_column = false self.inheritance_column = false
enum :type, { link: 0, photo: 1, video: 2, rich: 3 } enum :type, { link: 0, photo: 1, video: 2, rich: 3 }
@ -63,7 +68,7 @@ class PreviewCard < ApplicationRecord
convert_options: { all: '-quality 90 +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, convert_options: { all: '-quality 90 +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' },
validate_media_type: false validate_media_type: false
validates :url, presence: true, uniqueness: true, url: true validates :url, presence: true, uniqueness: true, url: true, length: { maximum: URL_CHARACTER_LIMIT }
validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES
validates_attachment_size :image, less_than: LIMIT validates_attachment_size :image, less_than: LIMIT
remotable_attachment :image, LIMIT remotable_attachment :image, LIMIT

7
app/services/fetch_link_card_service.rb
View file

@ -15,9 +15,6 @@ class FetchLinkCardService < BaseService
) )
}iox }iox
# URL size limit to safely store in PosgreSQL's unique indexes
BYTESIZE_LIMIT = 2692
def call(status) def call(status)
@status = status @status = status
@original_url = parse_urls @original_url = parse_urls
@ -32,7 +29,7 @@ class FetchLinkCardService < BaseService
end end
attach_card if @card&.persisted? attach_card if @card&.persisted?
rescue HTTP::Error, OpenSSL::SSL::SSLError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError, EncodingError => e rescue HTTP::Error, OpenSSL::SSL::SSLError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError, EncodingError, ActiveRecord::RecordInvalid => e
Rails.logger.debug { "Error fetching link #{@original_url}: #{e}" } Rails.logger.debug { "Error fetching link #{@original_url}: #{e}" }
nil nil
end end
@ -88,7 +85,7 @@ class FetchLinkCardService < BaseService
def bad_url?(uri) def bad_url?(uri)
# Avoid local instance URLs and invalid URLs # Avoid local instance URLs and invalid URLs
uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme) || uri.to_s.bytesize > BYTESIZE_LIMIT uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme)
end end
def mention_link?(anchor) def mention_link?(anchor)

18
spec/fixtures/requests/long_canonical_url.txt vendored Normal file
View file

@ -0,0 +1,18 @@
HTTP/1.1 200 OK
server: nginx
date: Thu, 13 Jun 2024 14:33:13 GMT
content-type: text/html; charset=utf-8
content-length: 3225
accept-ranges: bytes
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8">
<link rel="canonical" href="https://example.com/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa">
<title>Very long canonical URL</title>
</head>
<body>
<h2>We have very long URLs</h2>
</body>
</html>

22
spec/services/fetch_link_card_service_spec.rb
View file

@ -31,6 +31,7 @@ RSpec.describe FetchLinkCardService do
stub_request(:get, 'http://example.com/latin1_posing_as_utf8_recoverable').to_return(request_fixture('latin1_posing_as_utf8_recoverable.txt')) stub_request(:get, 'http://example.com/latin1_posing_as_utf8_recoverable').to_return(request_fixture('latin1_posing_as_utf8_recoverable.txt'))
stub_request(:get, 'http://example.com/aergerliche-umlaute').to_return(request_fixture('redirect_with_utf8_url.txt')) stub_request(:get, 'http://example.com/aergerliche-umlaute').to_return(request_fixture('redirect_with_utf8_url.txt'))
stub_request(:get, 'http://example.com/page_without_title').to_return(request_fixture('page_without_title.txt')) stub_request(:get, 'http://example.com/page_without_title').to_return(request_fixture('page_without_title.txt'))
stub_request(:get, 'http://example.com/long_canonical_url').to_return(request_fixture('long_canonical_url.txt'))
Rails.cache.write('oembed_endpoint:example.com', oembed_cache) if oembed_cache Rails.cache.write('oembed_endpoint:example.com', oembed_cache) if oembed_cache
@ -233,19 +234,6 @@ RSpec.describe FetchLinkCardService do
end end
end end
context 'with an URL too long for PostgreSQL unique indexes' do
let(:url) { "http://example.com/#{'a' * 2674}" }
let(:status) { Fabricate(:status, text: url) }
it 'does not fetch the URL' do
expect(a_request(:get, url)).to_not have_been_made
end
it 'does not create a preview card' do
expect(status.preview_card).to be_nil
end
end
context 'with a URL of a page with oEmbed support' do context 'with a URL of a page with oEmbed support' do
let(:html) { '<!doctype html><title>Hello world</title><link rel="alternate" type="application/json+oembed" href="http://example.com/oembed?url=http://example.com/html">' } let(:html) { '<!doctype html><title>Hello world</title><link rel="alternate" type="application/json+oembed" href="http://example.com/oembed?url=http://example.com/html">' }
let(:status) { Fabricate(:status, text: 'http://example.com/html') } let(:status) { Fabricate(:status, text: 'http://example.com/html') }
@ -296,6 +284,14 @@ RSpec.describe FetchLinkCardService do
end end
end end
end end
context 'with a URL of a page that includes a canonical URL too long for PostgreSQL unique indexes' do
let(:status) { Fabricate(:status, text: 'test http://example.com/long_canonical_url') }
it 'does not create a preview card' do
expect(status.preview_card).to be_nil
end
end
end end
context 'with a remote status' do context 'with a remote status' do