mirror of
https://github.com/mastodon/mastodon.git
synced 2024-08-20 21:08:15 -07:00
Fix performance of home feed regeneration (#12084)
Fetching statuses from all followed accounts at once takes too long within Postgres. Fetching them one by one and merging in Ruby could be a lot less resource-intensive Because the query for dynamically fetching the home timeline is so heavy, we can no longer offer it when the home timeline is missing
This commit is contained in:
Eugen Rochko
GitHub
parent
efda126914
commit
f665901e3c
13 changed files with 134 additions and 143 deletions
|
|
@ -13,7 +13,7 @@ class Api::V1::Timelines::HomeController < Api::BaseController
|
|||
render json: @statuses,
|
||||
each_serializer: REST::StatusSerializer,
|
||||
relationships: StatusRelationshipsPresenter.new(@statuses, current_user&.account_id),
|
||||
status: regeneration_in_progress? ? 206 : 200
|
||||
status: account_home_feed.regenerating? ? 206 : 200
|
||||
end
|
||||
|
||||
private
|
||||
|
|
@ -62,8 +62,4 @@ class Api::V1::Timelines::HomeController < Api::BaseController
|
|||
def pagination_since_id
|
||||
@statuses.first.id
|
||||
end
|
||||
|
||||
def regeneration_in_progress?
|
||||
Redis.current.exists("account:#{current_account.id}:regeneration")
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ export function expandTimeline(timelineId, path, params = {}, done = noOp) {
|
|||
api(getState).get(path, { params }).then(response => {
|
||||
const next = getLinks(response).refs.find(link => link.rel === 'next');
|
||||
dispatch(importFetchedStatuses(response.data));
|
||||
dispatch(expandTimelineSuccess(timelineId, response.data, next ? next.uri : null, response.code === 206, isLoadingRecent, isLoadingMore, isLoadingRecent && preferPendingItems));
|
||||
dispatch(expandTimelineSuccess(timelineId, response.data, next ? next.uri : null, response.status === 206, isLoadingRecent, isLoadingMore, isLoadingRecent && preferPendingItems));
|
||||
done();
|
||||
}).catch(error => {
|
||||
dispatch(expandTimelineFail(timelineId, error, isLoadingMore));
|
||||
|
|
|
|||
|
|
@ -1,17 +1,24 @@
|
|||
import React from 'react';
|
||||
import PropTypes from 'prop-types';
|
||||
import { FormattedMessage } from 'react-intl';
|
||||
import illustration from 'mastodon/../images/elephant_ui_disappointed.svg';
|
||||
import classNames from 'classnames';
|
||||
|
||||
const MissingIndicator = () => (
|
||||
<div className='regeneration-indicator missing-indicator'>
|
||||
<div>
|
||||
<div className='regeneration-indicator__figure' />
|
||||
const MissingIndicator = ({ fullPage }) => (
|
||||
<div className={classNames('regeneration-indicator', { 'regeneration-indicator--without-header': fullPage })}>
|
||||
<div className='regeneration-indicator__figure'>
|
||||
<img src={illustration} alt='' />
|
||||
</div>
|
||||
|
||||
<div className='regeneration-indicator__label'>
|
||||
<FormattedMessage id='missing_indicator.label' tagName='strong' defaultMessage='Not found' />
|
||||
<FormattedMessage id='missing_indicator.sublabel' defaultMessage='This resource could not be found' />
|
||||
</div>
|
||||
<div className='regeneration-indicator__label'>
|
||||
<FormattedMessage id='missing_indicator.label' tagName='strong' defaultMessage='Not found' />
|
||||
<FormattedMessage id='missing_indicator.sublabel' defaultMessage='This resource could not be found' />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
MissingIndicator.propTypes = {
|
||||
fullPage: PropTypes.bool,
|
||||
};
|
||||
|
||||
export default MissingIndicator;
|
||||
|
|
|
|||
18
app/javascript/mastodon/components/regeneration_indicator.js
Normal file
18
app/javascript/mastodon/components/regeneration_indicator.js
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
import React from 'react';
|
||||
import { FormattedMessage } from 'react-intl';
|
||||
import illustration from 'mastodon/../images/elephant_ui_working.svg';
|
||||
|
||||
const MissingIndicator = () => (
|
||||
<div className='regeneration-indicator'>
|
||||
<div className='regeneration-indicator__figure'>
|
||||
<img src={illustration} alt='' />
|
||||
</div>
|
||||
|
||||
<div className='regeneration-indicator__label'>
|
||||
<FormattedMessage id='regeneration_indicator.label' tagName='strong' defaultMessage='Loading…' />
|
||||
<FormattedMessage id='regeneration_indicator.sublabel' defaultMessage='Your home feed is being prepared!' />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
export default MissingIndicator;
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
import { debounce } from 'lodash';
|
||||
import React from 'react';
|
||||
import { FormattedMessage } from 'react-intl';
|
||||
import ImmutablePropTypes from 'react-immutable-proptypes';
|
||||
import PropTypes from 'prop-types';
|
||||
import StatusContainer from '../containers/status_container';
|
||||
import ImmutablePureComponent from 'react-immutable-pure-component';
|
||||
import LoadGap from './load_gap';
|
||||
import ScrollableList from './scrollable_list';
|
||||
import RegenerationIndicator from 'mastodon/components/regeneration_indicator';
|
||||
|
||||
export default class StatusList extends ImmutablePureComponent {
|
||||
|
||||
|
|
@ -81,18 +81,7 @@ export default class StatusList extends ImmutablePureComponent {
|
|||
const { isLoading, isPartial } = other;
|
||||
|
||||
if (isPartial) {
|
||||
return (
|
||||
<div className='regeneration-indicator'>
|
||||
<div>
|
||||
<div className='regeneration-indicator__figure' />
|
||||
|
||||
<div className='regeneration-indicator__label'>
|
||||
<FormattedMessage id='regeneration_indicator.label' tagName='strong' defaultMessage='Loading…' />
|
||||
<FormattedMessage id='regeneration_indicator.sublabel' defaultMessage='Your home feed is being prepared!' />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
return <RegenerationIndicator />;
|
||||
}
|
||||
|
||||
let scrollableContent = (isLoading || statusIds.size > 0) ? (
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import MissingIndicator from '../../components/missing_indicator';
|
|||
|
||||
const GenericNotFound = () => (
|
||||
<Column>
|
||||
<MissingIndicator />
|
||||
<MissingIndicator fullPage />
|
||||
</Column>
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -3127,37 +3127,27 @@ a.status-card.compact:hover {
|
|||
cursor: default;
|
||||
display: flex;
|
||||
flex: 1 1 auto;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 20px;
|
||||
|
||||
& > div {
|
||||
width: 100%;
|
||||
background: transparent;
|
||||
padding-top: 0;
|
||||
}
|
||||
|
||||
&__figure {
|
||||
background: url('../images/elephant_ui_working.svg') no-repeat center 0;
|
||||
width: 100%;
|
||||
height: 160px;
|
||||
background-size: contain;
|
||||
position: absolute;
|
||||
top: 50%;
|
||||
left: 50%;
|
||||
transform: translate(-50%, -50%);
|
||||
}
|
||||
|
||||
&.missing-indicator {
|
||||
padding-top: 20px + 48px;
|
||||
|
||||
.regeneration-indicator__figure {
|
||||
background-image: url('../images/elephant_ui_disappointed.svg');
|
||||
&,
|
||||
img {
|
||||
display: block;
|
||||
width: auto;
|
||||
height: 160px;
|
||||
margin: 0;
|
||||
}
|
||||
}
|
||||
|
||||
&--without-header {
|
||||
padding-top: 20px + 48px;
|
||||
}
|
||||
|
||||
&__label {
|
||||
margin-top: 200px;
|
||||
margin-top: 30px;
|
||||
|
||||
strong {
|
||||
display: block;
|
||||
|
|
|
|||
|
|
@ -3,9 +3,10 @@
|
|||
flex-direction: column;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
height: 100vh;
|
||||
background: $ui-base-color;
|
||||
|
||||
@media screen and (max-width: 920px) {
|
||||
background: darken($ui-base-color, 8%);
|
||||
display: block !important;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ class FeedManager
|
|||
|
||||
def filter?(timeline_type, status, receiver_id)
|
||||
if timeline_type == :home
|
||||
filter_from_home?(status, receiver_id)
|
||||
filter_from_home?(status, receiver_id, build_crutches(receiver_id, [status]))
|
||||
elsif timeline_type == :mentions
|
||||
filter_from_mentions?(status, receiver_id)
|
||||
else
|
||||
|
|
@ -29,6 +29,7 @@ class FeedManager
|
|||
|
||||
def push_to_home(account, status)
|
||||
return false unless add_to_feed(:home, account.id, status, account.user&.aggregates_reblogs?)
|
||||
|
||||
trim(:home, account.id)
|
||||
PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}") if push_update_required?("timeline:#{account.id}")
|
||||
true
|
||||
|
|
@ -36,6 +37,7 @@ class FeedManager
|
|||
|
||||
def unpush_from_home(account, status)
|
||||
return false unless remove_from_feed(:home, account.id, status, account.user&.aggregates_reblogs?)
|
||||
|
||||
redis.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s))
|
||||
true
|
||||
end
|
||||
|
|
@ -46,7 +48,9 @@ class FeedManager
|
|||
should_filter &&= !ListAccount.where(list_id: list.id, account_id: status.in_reply_to_account_id).exists?
|
||||
return false if should_filter
|
||||
end
|
||||
|
||||
return false unless add_to_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?)
|
||||
|
||||
trim(:list, list.id)
|
||||
PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}") if push_update_required?("timeline:list:#{list.id}")
|
||||
true
|
||||
|
|
@ -54,6 +58,7 @@ class FeedManager
|
|||
|
||||
def unpush_from_list(list, status)
|
||||
return false unless remove_from_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?)
|
||||
|
||||
redis.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s))
|
||||
true
|
||||
end
|
||||
|
|
@ -85,16 +90,21 @@ class FeedManager
|
|||
|
||||
def merge_into_timeline(from_account, into_account)
|
||||
timeline_key = key(:home, into_account.id)
|
||||
query = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4)
|
||||
aggregate = into_account.user&.aggregates_reblogs?
|
||||
query = from_account.statuses.where(visibility: [:public, :unlisted, :private]).includes(:preloadable_poll, reblog: :account).limit(FeedManager::MAX_ITEMS / 4)
|
||||
|
||||
if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4
|
||||
oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
|
||||
oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true).first.last.to_i
|
||||
query = query.where('id > ?', oldest_home_score)
|
||||
end
|
||||
|
||||
query.each do |status|
|
||||
next if status.direct_visibility? || status.limited_visibility? || filter?(:home, status, into_account)
|
||||
add_to_feed(:home, into_account.id, status, into_account.user&.aggregates_reblogs?)
|
||||
statuses = query.to_a
|
||||
crutches = build_crutches(into_account.id, statuses)
|
||||
|
||||
statuses.each do |status|
|
||||
next if filter_from_home?(status, into_account, crutches)
|
||||
|
||||
add_to_feed(:home, into_account.id, status, aggregate)
|
||||
end
|
||||
|
||||
trim(:home, into_account.id)
|
||||
|
|
@ -120,24 +130,35 @@ class FeedManager
|
|||
end
|
||||
|
||||
def populate_feed(account)
|
||||
added = 0
|
||||
limit = FeedManager::MAX_ITEMS / 2
|
||||
max_id = nil
|
||||
limit = FeedManager::MAX_ITEMS / 2
|
||||
aggregate = account.user&.aggregates_reblogs?
|
||||
timeline_key = key(:home, account.id)
|
||||
|
||||
loop do
|
||||
statuses = Status.as_home_timeline(account)
|
||||
.paginate_by_max_id(limit, max_id)
|
||||
account.statuses.where.not(visibility: :direct).limit(limit).each do |status|
|
||||
add_to_feed(:home, account.id, status, aggregate)
|
||||
end
|
||||
|
||||
break if statuses.empty?
|
||||
account.following.includes(:account_stat).find_each do |target_account|
|
||||
if redis.zcard(timeline_key) >= limit
|
||||
oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true).first.last.to_i
|
||||
last_status_score = Mastodon::Snowflake.id_at(account.last_status_at)
|
||||
|
||||
statuses.each do |status|
|
||||
next if filter_from_home?(status, account)
|
||||
added += 1 if add_to_feed(:home, account.id, status, account.user&.aggregates_reblogs?)
|
||||
# If the feed is full and this account has not posted more recently
|
||||
# than the last item on the feed, then we can skip the whole account
|
||||
# because none of its statuses would stay on the feed anyway
|
||||
next if last_status_score < oldest_home_score
|
||||
end
|
||||
|
||||
break unless added.zero?
|
||||
statuses = target_account.statuses.where(visibility: [:public, :unlisted, :private]).includes(:preloadable_poll, reblog: :account).limit(limit)
|
||||
crutches = build_crutches(account.id, statuses)
|
||||
|
||||
max_id = statuses.last.id
|
||||
statuses.each do |status|
|
||||
next if filter_from_home?(status, account, crutches)
|
||||
|
||||
add_to_feed(:home, account.id, status, aggregate)
|
||||
end
|
||||
|
||||
trim(:home, account.id)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -152,31 +173,33 @@ class FeedManager
|
|||
(context == :home ? Mute.where(account_id: receiver_id, target_account_id: account_ids).any? : Mute.where(account_id: receiver_id, target_account_id: account_ids, hide_notifications: true).any?)
|
||||
end
|
||||
|
||||
def filter_from_home?(status, receiver_id)
|
||||
def filter_from_home?(status, receiver_id, crutches)
|
||||
return false if receiver_id == status.account_id
|
||||
return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?)
|
||||
return true if phrase_filtered?(status, receiver_id, :home)
|
||||
|
||||
check_for_blocks = status.active_mentions.pluck(:account_id)
|
||||
check_for_blocks = crutches[:active_mentions][status.id] || []
|
||||
check_for_blocks.concat([status.account_id])
|
||||
|
||||
if status.reblog?
|
||||
check_for_blocks.concat([status.reblog.account_id])
|
||||
check_for_blocks.concat(status.reblog.active_mentions.pluck(:account_id))
|
||||
check_for_blocks.concat(crutches[:active_mentions][status.reblog_of_id] || [])
|
||||
end
|
||||
|
||||
return true if blocks_or_mutes?(receiver_id, check_for_blocks, :home)
|
||||
return true if check_for_blocks.any? { |target_account_id| crutches[:blocking][target_account_id] || crutches[:muting][target_account_id] }
|
||||
|
||||
if status.reply? && !status.in_reply_to_account_id.nil? # Filter out if it's a reply
|
||||
should_filter = !Follow.where(account_id: receiver_id, target_account_id: status.in_reply_to_account_id).exists? # and I'm not following the person it's a reply to
|
||||
should_filter = !crutches[:following][status.in_reply_to_account_id] # and I'm not following the person it's a reply to
|
||||
should_filter &&= receiver_id != status.in_reply_to_account_id # and it's not a reply to me
|
||||
should_filter &&= status.account_id != status.in_reply_to_account_id # and it's not a self-reply
|
||||
return should_filter
|
||||
|
||||
return !!should_filter
|
||||
elsif status.reblog? # Filter out a reblog
|
||||
should_filter = Follow.where(account_id: receiver_id, target_account_id: status.account_id, show_reblogs: false).exists? # if the reblogger's reblogs are suppressed
|
||||
should_filter ||= Block.where(account_id: status.reblog.account_id, target_account_id: receiver_id).exists? # or if the author of the reblogged status is blocking me
|
||||
should_filter ||= AccountDomainBlock.where(account_id: receiver_id, domain: status.reblog.account.domain).exists? # or the author's domain is blocked
|
||||
return should_filter
|
||||
should_filter = crutches[:hiding_reblogs][status.account_id] # if the reblogger's reblogs are suppressed
|
||||
should_filter ||= crutches[:blocked_by][status.reblog.account_id] # or if the author of the reblogged status is blocking me
|
||||
should_filter ||= crutches[:domain_blocking][status.reblog.account.domain] # or the author's domain is blocked
|
||||
|
||||
return !!should_filter
|
||||
end
|
||||
|
||||
false
|
||||
|
|
@ -308,4 +331,31 @@ class FeedManager
|
|||
|
||||
redis.zrem(timeline_key, status.id)
|
||||
end
|
||||
|
||||
def build_crutches(receiver_id, statuses)
|
||||
crutches = {}
|
||||
|
||||
crutches[:active_mentions] = Mention.active.where(status_id: statuses.flat_map { |s| [s.id, s.reblog_of_id] }.compact).pluck(:status_id, :account_id).each_with_object({}) { |(id, account_id), mapping| (mapping[id] ||= []).push(account_id) }
|
||||
|
||||
check_for_blocks = statuses.flat_map do |s|
|
||||
arr = crutches[:active_mentions][s.id] || []
|
||||
arr.concat([s.account_id])
|
||||
|
||||
if s.reblog?
|
||||
arr.concat([s.reblog.account_id])
|
||||
arr.concat(crutches[:active_mentions][s.reblog_of_id] || [])
|
||||
end
|
||||
|
||||
arr
|
||||
end
|
||||
|
||||
crutches[:following] = Follow.where(account_id: receiver_id, target_account_id: statuses.map(&:in_reply_to_account_id).compact).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
|
||||
crutches[:hiding_reblogs] = Follow.where(account_id: receiver_id, target_account_id: statuses.map { |s| s.account_id if s.reblog? }.compact, show_reblogs: false).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
|
||||
crutches[:blocking] = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
|
||||
crutches[:muting] = Mute.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
|
||||
crutches[:domain_blocking] = AccountDomainBlock.where(account_id: receiver_id, domain: statuses.map { |s| s.reblog&.account&.domain }.compact).pluck(:domain).each_with_object({}) { |domain, mapping| mapping[domain] = true }
|
||||
crutches[:blocked_by] = Block.where(target_account_id: receiver_id, account_id: statuses.map { |s| s.reblog&.account_id }.compact).pluck(:account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
|
||||
|
||||
crutches
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -7,19 +7,7 @@ class HomeFeed < Feed
|
|||
@account = account
|
||||
end
|
||||
|
||||
def get(limit, max_id = nil, since_id = nil, min_id = nil)
|
||||
if redis.exists("account:#{@account.id}:regeneration")
|
||||
from_database(limit, max_id, since_id, min_id)
|
||||
else
|
||||
super
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def from_database(limit, max_id, since_id, min_id)
|
||||
Status.as_home_timeline(@account)
|
||||
.paginate_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
|
||||
.reject { |status| FeedManager.instance.filter?(:home, status, @account.id) }
|
||||
def regenerating?
|
||||
redis.exists("account:#{@id}:regeneration")
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -282,10 +282,6 @@ class Status < ApplicationRecord
|
|||
where(language: nil).or where(language: account.chosen_languages)
|
||||
end
|
||||
|
||||
def as_home_timeline(account)
|
||||
where(account: [account] + account.following).where(visibility: [:public, :unlisted, :private])
|
||||
end
|
||||
|
||||
def as_public_timeline(account = nil, local_only = false)
|
||||
query = timeline_scope(local_only).without_replies
|
||||
|
||||
|
|
|
|||
|
|
@ -34,11 +34,10 @@ RSpec.describe HomeFeed, type: :model do
|
|||
Redis.current.set("account:#{account.id}:regeneration", true)
|
||||
end
|
||||
|
||||
it 'gets statuses with ids in the range from database' do
|
||||
it 'returns nothing' do
|
||||
results = subject.get(3)
|
||||
|
||||
expect(results.map(&:id)).to eq [10, 3, 2]
|
||||
expect(results.first.attributes.keys).to include('id', 'updated_at')
|
||||
expect(results.map(&:id)).to eq []
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -296,49 +296,6 @@ RSpec.describe Status, type: :model do
|
|||
end
|
||||
end
|
||||
|
||||
describe '.as_home_timeline' do
|
||||
let(:account) { Fabricate(:account) }
|
||||
let(:followed) { Fabricate(:account) }
|
||||
let(:not_followed) { Fabricate(:account) }
|
||||
|
||||
before do
|
||||
Fabricate(:follow, account: account, target_account: followed)
|
||||
|
||||
@self_status = Fabricate(:status, account: account, visibility: :public)
|
||||
@self_direct_status = Fabricate(:status, account: account, visibility: :direct)
|
||||
@followed_status = Fabricate(:status, account: followed, visibility: :public)
|
||||
@followed_direct_status = Fabricate(:status, account: followed, visibility: :direct)
|
||||
@not_followed_status = Fabricate(:status, account: not_followed, visibility: :public)
|
||||
|
||||
@results = Status.as_home_timeline(account)
|
||||
end
|
||||
|
||||
it 'includes statuses from self' do
|
||||
expect(@results).to include(@self_status)
|
||||
end
|
||||
|
||||
it 'does not include direct statuses from self' do
|
||||
expect(@results).to_not include(@self_direct_status)
|
||||
end
|
||||
|
||||
it 'includes statuses from followed' do
|
||||
expect(@results).to include(@followed_status)
|
||||
end
|
||||
|
||||
it 'does not include direct statuses mentioning recipient from followed' do
|
||||
Fabricate(:mention, account: account, status: @followed_direct_status)
|
||||
expect(@results).to_not include(@followed_direct_status)
|
||||
end
|
||||
|
||||
it 'does not include direct statuses not mentioning recipient from followed' do
|
||||
expect(@results).not_to include(@followed_direct_status)
|
||||
end
|
||||
|
||||
it 'does not include statuses from non-followed' do
|
||||
expect(@results).not_to include(@not_followed_status)
|
||||
end
|
||||
end
|
||||
|
||||
describe '.as_public_timeline' do
|
||||
it 'only includes statuses with public visibility' do
|
||||
public_status = Fabricate(:status, visibility: :public)
|
||||
|
|
|
|||
Loading…
Reference in a new issue