@ -63,20 +63,29 @@ const dbUrlToConfig = (dbUrl) => {
* @ param { Object . < string , any > } defaultConfig
* @ param { Object . < string , any > } defaultConfig
* @ param { string } redisUrl
* @ param { string } redisUrl
* /
* /
const redisUrlToClient = ( defaultConfig , redisUrl ) => { const redisUrlToClient = async ( defaultConfig , redisUrl ) => {
const config = defaultConfig ;
const config = defaultConfig ;
let client ;
if ( ! redisUrl ) {
if ( ! redisUrl ) {
return redis . createClient ( config ) ;
client = redis . createClient ( config ) ;
} else if ( redisUrl . startsWith ( 'unix://' ) ) {
client = redis . createClient ( Object . assign ( config , {
socket : {
path : redisUrl . slice ( 7 ) ,
} ,
} ) ) ;
} else {
client = redis . createClient ( Object . assign ( config , {
url : redisUrl ,
} ) ) ;
}
}
if ( redisUrl . startsWith ( 'unix://' ) ) {
client . on ( 'error' , ( err ) => log . error ( 'Redis Client Error!' , err ) ) ;
return redis . createClient ( redisUrl . slice ( 7 ) , config ) ;
await client . connect ( ) ;
}
return redis . createClient ( Object . assign ( config , {
return client ;
url : redisUrl ,
} ) ) ;
} ; } ;
const numWorkers = + process . env . STREAMING _CLUSTER _NUM || ( env === 'development' ? 1 : Math . max ( os . cpus ( ) . length - 1 , 1 ) ) ; const numWorkers = + process . env . STREAMING _CLUSTER _NUM || ( env === 'development' ? 1 : Math . max ( os . cpus ( ) . length - 1 , 1 ) ) ;
@ -102,7 +111,7 @@ const startMaster = () => {
log . warn ( ` Starting streaming API server master with ${ numWorkers } workers ` ) ;
log . warn ( ` Starting streaming API server master with ${ numWorkers } workers ` ) ;
} ; } ;
const startWorker = ( workerId ) => { const startWorker = async ( workerId ) => {
log . warn ( ` Starting worker ${ workerId } ` ) ;
log . warn ( ` Starting worker ${ workerId } ` ) ;
const pgConfigs = {
const pgConfigs = {
@ -139,9 +148,11 @@ const startWorker = (workerId) => {
const redisNamespace = process . env . REDIS _NAMESPACE || null ;
const redisNamespace = process . env . REDIS _NAMESPACE || null ;
const redisParams = {
const redisParams = {
socket : {
host : process . env . REDIS _HOST || '127.0.0.1' ,
host : process . env . REDIS _HOST || '127.0.0.1' ,
port : process . env . REDIS _PORT || 6379 ,
port : process . env . REDIS _PORT || 6379 ,
db : process . env . REDIS _DB || 0 ,
} ,
database : process . env . REDIS _DB || 0 ,
password : process . env . REDIS _PASSWORD || undefined ,
password : process . env . REDIS _PASSWORD || undefined ,
} ;
} ;
@ -151,25 +162,8 @@ const startWorker = (workerId) => {
const redisPrefix = redisNamespace ? ` ${ redisNamespace } : ` : '' ;
const redisPrefix = redisNamespace ? ` ${ redisNamespace } : ` : '' ;
const redisSubscribeClient = redisUrlToClient ( redisParams , process . env . REDIS _URL ) ;
const redisSubscribeClient = await redisUrlToClient ( redisParams , process . env . REDIS _URL ) ;
const redisClient = redisUrlToClient ( redisParams , process . env . REDIS _URL ) ;
const redisClient = await redisUrlToClient ( redisParams , process . env . REDIS _URL ) ;
/ * *
* @ type { Object . < string , Array . < function ( string ) : void >> }
* /
const subs = { } ;
redisSubscribeClient . on ( 'message' , ( channel , message ) => {
const callbacks = subs [ channel ] ;
log . silly ( ` New message on channel ${ channel } ` ) ;
if ( ! callbacks ) {
return ;
}
callbacks . forEach ( callback => callback ( message ) ) ;
} ) ;
/ * *
/ * *
* @ param { string [ ] } channels
* @ param { string [ ] } channels
@ -197,34 +191,16 @@ const startWorker = (workerId) => {
* /
* /
const subscribe = ( channel , callback ) => {
const subscribe = ( channel , callback ) => {
log . silly ( ` Adding listener for ${ channel } ` ) ;
log . silly ( ` Adding listener for ${ channel } ` ) ;
subs [ channel ] = subs [ channel ] || [ ] ;
if ( subs [ channel ] . length === 0 ) {
log . verbose ( ` Subscribe ${ channel } ` ) ;
redisSubscribeClient . subscribe ( channel ) ;
}
subs[ channel ] . push ( callback ) ;
redisSubscribeClient . subscribe ( channel , callback ) ;
} ;
} ;
/ * *
/ * *
* @ param { string } channel
* @ param { string } channel
* @ param { function ( string ) : void } callback
* /
* /
const unsubscribe = ( channel , callback ) => {
const unsubscribe = ( channel ) => {
log . silly ( ` Removing listener for ${ channel } ` ) ;
if ( ! subs [ channel ] ) {
return ;
}
subs [ channel ] = subs [ channel ] . filter ( item => item !== callback ) ;
if ( subs [ channel ] . length === 0 ) {
log . verbose ( ` Unsubscribe ${ channel } ` ) ;
redisSubscribeClient . unsubscribe ( channel ) ;
redisSubscribeClient . unsubscribe ( channel ) ;
delete subs [ channel ] ;
}
} ;
} ;
const FALSE _VALUES = [
const FALSE _VALUES = [
@ -366,7 +342,7 @@ const startWorker = (workerId) => {
const onlyMedia = isTruthy ( query . only _media ) ;
const onlyMedia = isTruthy ( query . only _media ) ;
const allowLocalOnly = isTruthy ( query . allow _local _only ) ;
const allowLocalOnly = isTruthy ( query . allow _local _only ) ;
switch ( path ) {
switch ( path ) {
case '/api/v1/streaming/user' :
case '/api/v1/streaming/user' :
return 'user' ;
return 'user' ;
case '/api/v1/streaming/user/notification' :
case '/api/v1/streaming/user/notification' :
@ -497,7 +473,7 @@ const startWorker = (workerId) => {
const listener = createSystemMessageListener ( req , {
const listener = createSystemMessageListener ( req , {
onKill ( ) {
onKill ( ) {
res . end ( ) ;
res . end ( ) ;
} ,
} ,
@ -549,7 +525,7 @@ const startWorker = (workerId) => {
} ;
} ;
/ * *
/ * *
* @ param { array }
* @ param { array } arr
* @ param { number = } shift
* @ param { number = } shift
* @ return { string }
* @ return { string }
* /
* /
@ -647,7 +623,15 @@ const startWorker = (workerId) => {
}
}
const queries = [
const queries = [
client . query ( ` SELECT 1 FROM blocks WHERE (account_id = $ 1 AND target_account_id IN ( ${ placeholders ( targetAccountIds , 2 ) } )) OR (account_id = $ 2 AND target_account_id = $ 1) UNION SELECT 1 FROM mutes WHERE account_id = $ 1 AND target_account_id IN ( ${ placeholders ( targetAccountIds , 2 ) } ) ` , [ req . accountId , unpackedPayload . account . id ] . concat ( targetAccountIds ) ) ,
client . query ( ` SELECT 1
FROM blocks
WHERE ( account _id = $1 AND target _account _id IN ( $ { placeholders ( targetAccountIds , 2 ) } ) )
OR ( account _id = $2 AND target _account _id = $1 )
UNION
SELECT 1
FROM mutes
WHERE account _id = $1
AND target _account _id IN ( $ { placeholders ( targetAccountIds , 2 ) } ) ` , [req.accountId, unpackedPayload.account.id].concat(targetAccountIds)),
] ;
] ;
if ( accountDomain ) {
if ( accountDomain ) {
@ -710,12 +694,12 @@ const startWorker = (workerId) => {
/ * *
/ * *
* @ param { any } req
* @ param { any } req
* @ param { function ( ) : void } [ closeHandler ]
* @ param { function ( ) : void } [ closeHandler ]
* @ return { function ( string [ ] , function ( string ): void ) }
* @ return { function ( string [ ] : void }
* /
* /
const streamHttpEnd = ( req , closeHandler = undefined ) => ( ids , listener ) => {
const streamHttpEnd = ( req , closeHandler = undefined ) => ( ids => {
req . on ( 'close' , ( ) => {
req . on ( 'close' , ( ) => {
ids . forEach ( id => {
ids . forEach ( id => {
unsubscribe ( id , listener );
unsubscribe ( id ;
} ) ;
} ) ;
if ( closeHandler ) {
if ( closeHandler ) {
@ -805,7 +789,7 @@ const startWorker = (workerId) => {
* @ return { Promise . < { channelIds : string [ ] , options : { needsFiltering : boolean } } > }
* @ return { Promise . < { channelIds : string [ ] , options : { needsFiltering : boolean } } > }
* /
* /
const channelNameToIds = ( req , name , params ) => new Promise ( ( resolve , reject ) => {
const channelNameToIds = ( req , name , params ) => new Promise ( ( resolve , reject ) => {
switch ( name ) {
switch ( name ) {
case 'user' :
case 'user' :
resolve ( {
resolve ( {
channelIds : channelsForUserStream ( req ) ,
channelIds : channelsForUserStream ( req ) ,
@ -949,14 +933,16 @@ const startWorker = (workerId) => {
* @ param { StreamParams } params
* @ param { StreamParams } params
* /
* /
const subscribeWebsocketToChannel = ( { socket , request , subscriptions } , channelName , params ) =>
const subscribeWebsocketToChannel = ( { socket , request , subscriptions } , channelName , params ) =>
checkScopes ( request , channelName ) . then ( ( ) => channelNameToIds ( request , channelName , params ) ) . then ( ( { channelIds , options } ) => {
checkScopes ( request , channelName ) . then ( ( ) => channelNameToIds ( request , channelName , params ) ) . then ( ( {
channelIds ,
options ,
} ) => {
if ( subscriptions [ channelIds . join ( ';' ) ] ) {
if ( subscriptions [ channelIds . join ( ';' ) ] ) {
return ;
return ;
}
}
const onSend = streamToWs ( request , socket , streamNameFromChannelName ( channelName , params ) ) ;
const onSend = streamToWs ( request , socket , streamNameFromChannelName ( channelName , params ) ) ;
const stopHeartbeat = subscriptionHeartbeat ( channelIds ) ;
const stopHeartbeat = subscriptionHeartbeat ( channelIds ) ;
const listener = streamFrom ( channelIds , request , onSend , undefined , options . needsFiltering , options . allowLocalOnly ) ;
const listener = streamFrom ( channelIds , request , onSend , undefined , options . needsFiltering , options . allowLocalOnly ) ;
subscriptions [ channelIds . join ( ';' ) ] = {
subscriptions [ channelIds . join ( ';' ) ] = {
@ -1005,7 +991,7 @@ const startWorker = (workerId) => {
const listener = createSystemMessageListener ( request , {
const listener = createSystemMessageListener ( request , {
onKill ( ) {
onKill ( ) {
socket . close ( ) ;
socket . close ( ) ;
} ,
} ,
@ -1015,7 +1001,8 @@ const startWorker = (workerId) => {
subscriptions [ systemChannelId ] = {
subscriptions [ systemChannelId ] = {
listener ,
listener ,
stopHeartbeat : ( ) => { } ,
stopHeartbeat : ( ) => {
} ,
} ;
} ;
} ;
} ;
Claire
3 years ago