teh/mastodon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 10

Finish update of node-redis (#17183)

Browse source 
* fix streaming redis client

* use console.error instead of console.log

* follow node-redis migration guide

https://github.com/redis/node-redis/blob/master/docs/v3-to-v4.md

* fix config options for node-redis

* keep indentation

* Update streaming/index.js

Co-authored-by: Yamagishi Kazutoshi <ykzts@desire.sh>

Co-authored-by: Yamagishi Kazutoshi <ykzts@desire.sh>
This commit is contained in:
Lerk 2021-12-25 21:55:06 +00:00 committed by GitHub
parent fad37dd1bc
commit 4d1eaf3e6e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 61 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

110
streaming/index.js
View file

@ -63,20 +63,29 @@ const dbUrlToConfig = (dbUrl) => {
* @param {Object.<string, any>} defaultConfig * @param {Object.<string, any>} defaultConfig
* @param {string} redisUrl * @param {string} redisUrl
*/ */
const redisUrlToClient = (defaultConfig, redisUrl) => { const redisUrlToClient = async (defaultConfig, redisUrl) => {
const config = defaultConfig; const config = defaultConfig;
let client;
if (!redisUrl) { if (!redisUrl) {
return redis.createClient(config); client = redis.createClient(config);
} } else if (redisUrl.startsWith('unix://')) {
client = redis.createClient(Object.assign(config, {
if (redisUrl.startsWith('unix://')) { socket: {
return redis.createClient(redisUrl.slice(7), config); path: redisUrl.slice(7),
} },
}));
return redis.createClient(Object.assign(config, { } else {
client = redis.createClient(Object.assign(config, {
url: redisUrl, url: redisUrl,
})); }));
}
client.on('error', (err) => log.error('Redis Client Error!', err));
await client.connect();
return client;
}; };
const numWorkers = +process.env.STREAMING_CLUSTER_NUM || (env === 'development' ? 1 : Math.max(os.cpus().length - 1, 1)); const numWorkers = +process.env.STREAMING_CLUSTER_NUM || (env === 'development' ? 1 : Math.max(os.cpus().length - 1, 1));
@ -102,7 +111,7 @@ const startMaster = () => {
log.warn(`Starting streaming API server master with ${numWorkers} workers`); log.warn(`Starting streaming API server master with ${numWorkers} workers`);
}; };
const startWorker = (workerId) => { const startWorker = async (workerId) => {
log.warn(`Starting worker ${workerId}`); log.warn(`Starting worker ${workerId}`);
const pgConfigs = { const pgConfigs = {
@ -139,9 +148,11 @@ const startWorker = (workerId) => {
const redisNamespace = process.env.REDIS_NAMESPACE || null; const redisNamespace = process.env.REDIS_NAMESPACE || null;
const redisParams = { const redisParams = {
socket: {
host: process.env.REDIS_HOST || '127.0.0.1', host: process.env.REDIS_HOST || '127.0.0.1',
port: process.env.REDIS_PORT || 6379, port: process.env.REDIS_PORT || 6379,
db: process.env.REDIS_DB || 0, },
database: process.env.REDIS_DB || 0,
password: process.env.REDIS_PASSWORD || undefined, password: process.env.REDIS_PASSWORD || undefined,
}; };
@ -151,25 +162,8 @@ const startWorker = (workerId) => {
const redisPrefix = redisNamespace ? `${redisNamespace}:` : ''; const redisPrefix = redisNamespace ? `${redisNamespace}:` : '';
const redisSubscribeClient = redisUrlToClient(redisParams, process.env.REDIS_URL); const redisSubscribeClient = await redisUrlToClient(redisParams, process.env.REDIS_URL);
const redisClient = redisUrlToClient(redisParams, process.env.REDIS_URL); const redisClient = await redisUrlToClient(redisParams, process.env.REDIS_URL);
/**
* @type {Object.<string, Array.<function(string): void>>}
*/
const subs = {};
redisSubscribeClient.on('message', (channel, message) => {
const callbacks = subs[channel];
log.silly(`New message on channel ${channel}`);
if (!callbacks) {
return;
}
callbacks.forEach(callback => callback(message));
});
/** /**
* @param {string[]} channels * @param {string[]} channels
@ -197,34 +191,16 @@ const startWorker = (workerId) => {
*/ */
const subscribe = (channel, callback) => { const subscribe = (channel, callback) => {
log.silly(`Adding listener for ${channel}`); log.silly(`Adding listener for ${channel}`);
subs[channel] = subs[channel] || [];
if (subs[channel].length === 0) { redisSubscribeClient.subscribe(channel, callback);
log.verbose(`Subscribe ${channel}`);
redisSubscribeClient.subscribe(channel);
}
subs[channel].push(callback);
}; };
/** /**
* @param {string} channel * @param {string} channel
* @param {function(string): void} callback
*/ */
const unsubscribe = (channel, callback) => { const unsubscribe = (channel) => {
log.silly(`Removing listener for ${channel}`);
if (!subs[channel]) {
return;
}
subs[channel] = subs[channel].filter(item => item !== callback);
if (subs[channel].length === 0) {
log.verbose(`Unsubscribe ${channel}`);
redisSubscribeClient.unsubscribe(channel); redisSubscribeClient.unsubscribe(channel);
delete subs[channel];
}
}; };
const FALSE_VALUES = [ const FALSE_VALUES = [
@ -365,7 +341,7 @@ const startWorker = (workerId) => {
const { path, query } = req; const { path, query } = req;
const onlyMedia = isTruthy(query.only_media); const onlyMedia = isTruthy(query.only_media);
switch(path) { switch (path) {
case '/api/v1/streaming/user': case '/api/v1/streaming/user':
return 'user'; return 'user';
case '/api/v1/streaming/user/notification': case '/api/v1/streaming/user/notification':
@ -496,7 +472,7 @@ const startWorker = (workerId) => {
const listener = createSystemMessageListener(req, { const listener = createSystemMessageListener(req, {
onKill () { onKill() {
res.end(); res.end();
}, },
@ -548,7 +524,7 @@ const startWorker = (workerId) => {
}; };
/** /**
* @param {array} * @param {array} arr
* @param {number=} shift * @param {number=} shift
* @return {string} * @return {string}
*/ */
@ -639,7 +615,15 @@ const startWorker = (workerId) => {
} }
const queries = [ const queries = [
client.query(`SELECT 1 FROM blocks WHERE (account_id = $1 AND target_account_id IN (${placeholders(targetAccountIds, 2)})) OR (account_id = $2 AND target_account_id = $1) UNION SELECT 1 FROM mutes WHERE account_id = $1 AND target_account_id IN (${placeholders(targetAccountIds, 2)})`, [req.accountId, unpackedPayload.account.id].concat(targetAccountIds)), client.query(`SELECT 1
FROM blocks
WHERE (account_id = $1 AND target_account_id IN (${placeholders(targetAccountIds, 2)}))
OR (account_id = $2 AND target_account_id = $1)
UNION
SELECT 1
FROM mutes
WHERE account_id = $1
AND target_account_id IN (${placeholders(targetAccountIds, 2)})`, [req.accountId, unpackedPayload.account.id].concat(targetAccountIds)),
]; ];
if (accountDomain) { if (accountDomain) {
@ -702,12 +686,12 @@ const startWorker = (workerId) => {
/** /**
* @param {any} req * @param {any} req
* @param {function(): void} [closeHandler] * @param {function(): void} [closeHandler]
* @return {function(string[], function(string): void)} * @return {function(string[]): void}
*/ */
const streamHttpEnd = (req, closeHandler = undefined) => (ids, listener) => { const streamHttpEnd = (req, closeHandler = undefined) => (ids) => {
req.on('close', () => { req.on('close', () => {
ids.forEach(id => { ids.forEach(id => {
unsubscribe(id, listener); unsubscribe(id);
}); });
if (closeHandler) { if (closeHandler) {
@ -797,7 +781,7 @@ const startWorker = (workerId) => {
* @return {Promise.<{ channelIds: string[], options: { needsFiltering: boolean } }>} * @return {Promise.<{ channelIds: string[], options: { needsFiltering: boolean } }>}
*/ */
const channelNameToIds = (req, name, params) => new Promise((resolve, reject) => { const channelNameToIds = (req, name, params) => new Promise((resolve, reject) => {
switch(name) { switch (name) {
case 'user': case 'user':
resolve({ resolve({
channelIds: channelsForUserStream(req), channelIds: channelsForUserStream(req),
@ -927,7 +911,10 @@ const startWorker = (workerId) => {
* @param {StreamParams} params * @param {StreamParams} params
*/ */
const subscribeWebsocketToChannel = ({ socket, request, subscriptions }, channelName, params) => const subscribeWebsocketToChannel = ({ socket, request, subscriptions }, channelName, params) =>
checkScopes(request, channelName).then(() => channelNameToIds(request, channelName, params)).then(({ channelIds, options }) => { checkScopes(request, channelName).then(() => channelNameToIds(request, channelName, params)).then(({
channelIds,
options,
}) => {
if (subscriptions[channelIds.join(';')]) { if (subscriptions[channelIds.join(';')]) {
return; return;
} }
@ -982,7 +969,7 @@ const startWorker = (workerId) => {
const listener = createSystemMessageListener(request, { const listener = createSystemMessageListener(request, {
onKill () { onKill() {
socket.close(); socket.close();
}, },
@ -992,7 +979,8 @@ const startWorker = (workerId) => {
subscriptions[systemChannelId] = { subscriptions[systemChannelId] = {
listener, listener,
stopHeartbeat: () => {}, stopHeartbeat: () => {
},
}; };
}; };