mirror of
https://github.com/mastodon/mastodon.git
synced 2024-08-20 21:08:15 -07:00
Streaming: fetch keyword filters on connection
This commit is contained in:
Emelia Smith
parent
00cb4a0313
commit
4bd2728d7e
1 changed files with 106 additions and 66 deletions
|
|
@ -35,6 +35,29 @@ dotenv.config({
|
||||||
|
|
||||||
initializeLogLevel(process.env, environment);
|
initializeLogLevel(process.env, environment);
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef Filter
|
||||||
|
* @property {number} id
|
||||||
|
* @property {string} title
|
||||||
|
* @property {string[]} context
|
||||||
|
* @property {Date} expires_at
|
||||||
|
* @property {'warn' | 'hide'} filter_action
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef KeywordFilter
|
||||||
|
* @property {Array<[string, boolean]>} keywords
|
||||||
|
* @property {Date} expires_at
|
||||||
|
* @property {Filter} filter
|
||||||
|
* @property {RegExp} regexp
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef Filters
|
||||||
|
* @type {Object<string, KeywordFilter>}
|
||||||
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Declares the result type for accountFromToken / accountFromRequest.
|
* Declares the result type for accountFromToken / accountFromRequest.
|
||||||
*
|
*
|
||||||
|
|
@ -46,6 +69,7 @@ initializeLogLevel(process.env, environment);
|
||||||
* @property {string} accountId
|
* @property {string} accountId
|
||||||
* @property {string[]} chosenLanguages
|
* @property {string[]} chosenLanguages
|
||||||
* @property {string} deviceId
|
* @property {string} deviceId
|
||||||
|
* @property {Filters} cachedFilters
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -537,12 +561,15 @@ const startServer = async () => {
|
||||||
req.chosenLanguages = result.rows[0].chosen_languages;
|
req.chosenLanguages = result.rows[0].chosen_languages;
|
||||||
req.deviceId = result.rows[0].device_id;
|
req.deviceId = result.rows[0].device_id;
|
||||||
|
|
||||||
|
req.cachedFilters = await fetchFilters(req.accountId);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
accessTokenId: result.rows[0].id,
|
accessTokenId: result.rows[0].id,
|
||||||
scopes: result.rows[0].scopes.split(' '),
|
scopes: result.rows[0].scopes.split(' '),
|
||||||
accountId: result.rows[0].account_id,
|
accountId: result.rows[0].account_id,
|
||||||
chosenLanguages: result.rows[0].chosen_languages,
|
chosenLanguages: result.rows[0].chosen_languages,
|
||||||
deviceId: result.rows[0].device_id
|
deviceId: result.rows[0].device_id,
|
||||||
|
cachedFilters: req.cachedFilters
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -636,6 +663,76 @@ const startServer = async () => {
|
||||||
reject(new AuthenticationError('Access token does not have the required scopes'));
|
reject(new AuthenticationError('Access token does not have the required scopes'));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
|
const FILTER_ACTIONS = [
|
||||||
|
'warn',
|
||||||
|
'hide'
|
||||||
|
];
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {string} accountId
|
||||||
|
* @returns {Promise<Filters>}
|
||||||
|
*/
|
||||||
|
const fetchFilters = async (accountId) => {
|
||||||
|
const results = await pgPool.query('SELECT filter.id AS id, filter.phrase AS title, filter.context AS context, filter.expires_at AS expires_at, filter.action AS filter_action, keyword.keyword AS keyword, keyword.whole_word AS whole_word FROM custom_filter_keywords keyword JOIN custom_filters filter ON keyword.custom_filter_id = filter.id WHERE filter.account_id = $1 AND (filter.expires_at IS NULL OR filter.expires_at > NOW())', [accountId]);
|
||||||
|
|
||||||
|
if (results.rows.length === 0) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
const filters = results.rows.reduce((filters, filter) => {
|
||||||
|
if (filters[filter.id]) {
|
||||||
|
filters[filter.id].keywords.push([filter.keyword, filter.whole_word]);
|
||||||
|
} else {
|
||||||
|
filters[filter.id] = {
|
||||||
|
keywords: [[filter.keyword, filter.whole_word]],
|
||||||
|
expires_at: filter.expires_at,
|
||||||
|
filter: {
|
||||||
|
id: filter.id,
|
||||||
|
title: filter.title,
|
||||||
|
context: filter.context,
|
||||||
|
expires_at: filter.expires_at,
|
||||||
|
// filter.filter_action is the value from the
|
||||||
|
// custom_filters.action database column, it is an integer
|
||||||
|
// representing a value in an enum defined by Ruby on Rails:
|
||||||
|
//
|
||||||
|
// enum { warn: 0, hide: 1 }
|
||||||
|
filter_action: FILTER_ACTIONS[filter.filter_action],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return filters;
|
||||||
|
}, {});
|
||||||
|
|
||||||
|
// Construct the regular expressions for the custom filters: This
|
||||||
|
// needs to be done in a separate loop as the database returns one
|
||||||
|
// filterRow per keyword, so we need all the keywords before
|
||||||
|
// constructing the regular expression
|
||||||
|
// @ts-ignore
|
||||||
|
Object.keys(filters).forEach((id) => {
|
||||||
|
// @ts-ignore
|
||||||
|
filters[id].regexp = new RegExp(filters[id].keywords.map(([keyword, whole_word]) => {
|
||||||
|
let expr = keyword.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||||
|
|
||||||
|
if (whole_word) {
|
||||||
|
if (/^[\w]/.test(expr)) {
|
||||||
|
expr = `\\b${expr}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (/[\w]$/.test(expr)) {
|
||||||
|
expr = `${expr}\\b`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return expr;
|
||||||
|
}).join('|'), 'i');
|
||||||
|
});
|
||||||
|
|
||||||
|
return filters;
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @typedef SystemMessageHandlers
|
* @typedef SystemMessageHandlers
|
||||||
* @property {function(): void} onKill
|
* @property {function(): void} onKill
|
||||||
|
|
@ -661,7 +758,12 @@ const startServer = async () => {
|
||||||
eventHandlers.onKill();
|
eventHandlers.onKill();
|
||||||
} else if (event === 'filters_changed') {
|
} else if (event === 'filters_changed') {
|
||||||
req.log.debug(`Invalidating filters cache for ${req.accountId}`);
|
req.log.debug(`Invalidating filters cache for ${req.accountId}`);
|
||||||
req.cachedFilters = null;
|
|
||||||
|
fetchFilters(req.accountId).then((filters) => {
|
||||||
|
req.cachedFilters = filters;
|
||||||
|
}).catch((err) => {
|
||||||
|
req.log.error({ err }, 'Error refreshing filters');
|
||||||
|
});
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
@ -865,12 +967,6 @@ const startServer = async () => {
|
||||||
queries.push(client.query('SELECT 1 FROM account_domain_blocks WHERE account_id = $1 AND domain = $2', [req.accountId, accountDomain]));
|
queries.push(client.query('SELECT 1 FROM account_domain_blocks WHERE account_id = $1 AND domain = $2', [req.accountId, accountDomain]));
|
||||||
}
|
}
|
||||||
|
|
||||||
// @ts-ignore
|
|
||||||
if (!payload.filtered && !req.cachedFilters) {
|
|
||||||
// @ts-ignore
|
|
||||||
queries.push(client.query('SELECT filter.id AS id, filter.phrase AS title, filter.context AS context, filter.expires_at AS expires_at, filter.action AS filter_action, keyword.keyword AS keyword, keyword.whole_word AS whole_word FROM custom_filter_keywords keyword JOIN custom_filters filter ON keyword.custom_filter_id = filter.id WHERE filter.account_id = $1 AND (filter.expires_at IS NULL OR filter.expires_at > NOW())', [req.accountId]));
|
|
||||||
}
|
|
||||||
|
|
||||||
Promise.all(queries).then(values => {
|
Promise.all(queries).then(values => {
|
||||||
releasePgConnection();
|
releasePgConnection();
|
||||||
|
|
||||||
|
|
@ -888,67 +984,10 @@ const startServer = async () => {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handling for constructing the custom filters and caching them on the request
|
|
||||||
// TODO: Move this logic out of the message handling lifecycle
|
|
||||||
// @ts-ignore
|
|
||||||
if (!req.cachedFilters) {
|
|
||||||
const filterRows = values[accountDomain ? 2 : 1].rows;
|
|
||||||
|
|
||||||
// @ts-ignore
|
|
||||||
req.cachedFilters = filterRows.reduce((cache, filter) => {
|
|
||||||
if (cache[filter.id]) {
|
|
||||||
cache[filter.id].keywords.push([filter.keyword, filter.whole_word]);
|
|
||||||
} else {
|
|
||||||
cache[filter.id] = {
|
|
||||||
keywords: [[filter.keyword, filter.whole_word]],
|
|
||||||
expires_at: filter.expires_at,
|
|
||||||
filter: {
|
|
||||||
id: filter.id,
|
|
||||||
title: filter.title,
|
|
||||||
context: filter.context,
|
|
||||||
expires_at: filter.expires_at,
|
|
||||||
// filter.filter_action is the value from the
|
|
||||||
// custom_filters.action database column, it is an integer
|
|
||||||
// representing a value in an enum defined by Ruby on Rails:
|
|
||||||
//
|
|
||||||
// enum { warn: 0, hide: 1 }
|
|
||||||
filter_action: ['warn', 'hide'][filter.filter_action],
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return cache;
|
|
||||||
}, {});
|
|
||||||
|
|
||||||
// Construct the regular expressions for the custom filters: This
|
|
||||||
// needs to be done in a separate loop as the database returns one
|
|
||||||
// filterRow per keyword, so we need all the keywords before
|
|
||||||
// constructing the regular expression
|
|
||||||
// @ts-ignore
|
|
||||||
Object.keys(req.cachedFilters).forEach((key) => {
|
|
||||||
// @ts-ignore
|
|
||||||
req.cachedFilters[key].regexp = new RegExp(req.cachedFilters[key].keywords.map(([keyword, whole_word]) => {
|
|
||||||
let expr = keyword.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
||||||
|
|
||||||
if (whole_word) {
|
|
||||||
if (/^[\w]/.test(expr)) {
|
|
||||||
expr = `\\b${expr}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (/[\w]$/.test(expr)) {
|
|
||||||
expr = `${expr}\\b`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return expr;
|
|
||||||
}).join('|'), 'i');
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Apply cachedFilters against the payload, constructing a
|
// Apply cachedFilters against the payload, constructing a
|
||||||
// `filter_results` array of FilterResult entities
|
// `filter_results` array of FilterResult entities
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
if (req.cachedFilters) {
|
if (Object.keys(req.cachedFilters).length) {
|
||||||
const status = payload;
|
const status = payload;
|
||||||
// TODO: Calculate searchableContent in Ruby on Rails:
|
// TODO: Calculate searchableContent in Ruby on Rails:
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
|
|
@ -974,6 +1013,7 @@ const startServer = async () => {
|
||||||
// null as we only are only applying the keyword-based custom
|
// null as we only are only applying the keyword-based custom
|
||||||
// filters, not the status-based custom filters.
|
// filters, not the status-based custom filters.
|
||||||
// https://docs.joinmastodon.org/entities/FilterResult/
|
// https://docs.joinmastodon.org/entities/FilterResult/
|
||||||
|
// @ts-ignore
|
||||||
results.push({
|
results.push({
|
||||||
filter: cachedFilter.filter,
|
filter: cachedFilter.filter,
|
||||||
keyword_matches,
|
keyword_matches,
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue