2020-08-11 19:24:59 +03:00
// @ts-check
2017-06-26 05:49:39 +03:00
const os = require ( 'os' ) ;
const throng = require ( 'throng' ) ;
const dotenv = require ( 'dotenv' ) ;
const express = require ( 'express' ) ;
const http = require ( 'http' ) ;
const redis = require ( 'redis' ) ;
const pg = require ( 'pg' ) ;
const log = require ( 'npmlog' ) ;
const url = require ( 'url' ) ;
const uuid = require ( 'uuid' ) ;
2018-08-24 19:16:53 +03:00
const fs = require ( 'fs' ) ;
2021-03-24 10:37:41 +02:00
const WebSocket = require ( 'ws' ) ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 10:42:13 +03:00
const { JSDOM } = require ( 'jsdom' ) ;
2017-05-20 18:31:47 +03:00
const env = process . env . NODE _ENV || 'development' ;
2020-08-11 19:24:59 +03:00
const alwaysRequireAuth = process . env . LIMITED _FEDERATION _MODE === 'true' || process . env . WHITELIST _MODE === 'true' || process . env . AUTHORIZED _FETCH === 'true' ;
2017-02-02 17:11:36 +02:00
dotenv . config ( {
2017-05-20 18:31:47 +03:00
path : env === 'production' ? '.env.production' : '.env' ,
} ) ;
2017-02-02 02:31:09 +02:00
2017-05-28 17:25:26 +03:00
log . level = process . env . LOG _LEVEL || 'verbose' ;
2020-08-11 19:24:59 +03:00
/ * *
* @ param { string } dbUrl
* @ return { Object . < string , any > }
* /
2017-05-04 00:18:13 +03:00
const dbUrlToConfig = ( dbUrl ) => {
if ( ! dbUrl ) {
2017-05-20 18:31:47 +03:00
return { } ;
2017-05-04 00:18:13 +03:00
}
2019-03-10 17:00:54 +02:00
const params = url . parse ( dbUrl , true ) ;
2017-05-20 18:31:47 +03:00
const config = { } ;
2017-05-04 16:53:44 +03:00
if ( params . auth ) {
2017-05-20 18:31:47 +03:00
[ config . user , config . password ] = params . auth . split ( ':' ) ;
2017-05-04 16:53:44 +03:00
}
if ( params . hostname ) {
2017-05-20 18:31:47 +03:00
config . host = params . hostname ;
2017-05-04 16:53:44 +03:00
}
if ( params . port ) {
2017-05-20 18:31:47 +03:00
config . port = params . port ;
2017-05-04 00:18:13 +03:00
}
2017-05-04 16:53:44 +03:00
if ( params . pathname ) {
2017-05-20 18:31:47 +03:00
config . database = params . pathname . split ( '/' ) [ 1 ] ;
2017-05-04 16:53:44 +03:00
}
2017-05-20 18:31:47 +03:00
const ssl = params . query && params . query . ssl ;
2017-05-20 22:06:09 +03:00
2019-03-10 17:00:54 +02:00
if ( ssl && ssl === 'true' || ssl === '1' ) {
config . ssl = true ;
2017-05-04 16:53:44 +03:00
}
2017-05-20 18:31:47 +03:00
return config ;
} ;
2017-05-04 00:18:13 +03:00
2020-08-11 19:24:59 +03:00
/ * *
* @ param { Object . < string , any > } defaultConfig
* @ param { string } redisUrl
* /
2021-12-25 23:55:06 +02:00
const redisUrlToClient = async ( defaultConfig , redisUrl ) => {
2017-05-20 22:06:09 +03:00
const config = defaultConfig ;
2021-12-25 23:55:06 +02:00
let client ;
2017-05-20 22:06:09 +03:00
if ( ! redisUrl ) {
2021-12-25 23:55:06 +02:00
client = redis . createClient ( config ) ;
} else if ( redisUrl . startsWith ( 'unix://' ) ) {
client = redis . createClient ( Object . assign ( config , {
socket : {
path : redisUrl . slice ( 7 ) ,
} ,
} ) ) ;
} else {
client = redis . createClient ( Object . assign ( config , {
url : redisUrl ,
} ) ) ;
2017-05-20 22:06:09 +03:00
}
2021-12-25 23:55:06 +02:00
client . on ( 'error' , ( err ) => log . error ( 'Redis Client Error!' , err ) ) ;
await client . connect ( ) ;
2017-05-20 22:06:09 +03:00
2021-12-25 23:55:06 +02:00
return client ;
2017-05-20 22:06:09 +03:00
} ;
2017-05-28 17:25:26 +03:00
const numWorkers = + process . env . STREAMING _CLUSTER _NUM || ( env === 'development' ? 1 : Math . max ( os . cpus ( ) . length - 1 , 1 ) ) ;
2017-05-04 00:18:13 +03:00
2020-09-22 16:30:41 +03:00
/ * *
* @ param { string } json
2022-02-16 15:37:26 +02:00
* @ param { any } req
2020-09-22 16:30:41 +03:00
* @ return { Object . < string , any > | null }
* /
2022-02-16 15:37:26 +02:00
const parseJSON = ( json , req ) => {
2020-09-22 16:30:41 +03:00
try {
return JSON . parse ( json ) ;
} catch ( err ) {
2022-02-16 15:37:26 +02:00
if ( req . accountId ) {
log . warn ( req . requestId , ` Error parsing message from user ${ req . accountId } : ${ err } ` ) ;
} else {
log . silly ( req . requestId , ` Error parsing message from ${ req . remoteAddress } : ${ err } ` ) ;
}
2020-09-22 16:30:41 +03:00
return null ;
}
} ;
2017-05-28 17:25:26 +03:00
const startMaster = ( ) => {
2018-08-24 19:16:53 +03:00
if ( ! process . env . SOCKET && process . env . PORT && isNaN ( + process . env . PORT ) ) {
log . warn ( 'UNIX domain socket is now supported by using SOCKET. Please migrate from PORT hack.' ) ;
}
2018-10-20 03:25:25 +03:00
2021-05-02 00:19:18 +03:00
log . warn ( ` Starting streaming API server master with ${ numWorkers } workers ` ) ;
2017-05-28 17:25:26 +03:00
} ;
2017-05-04 00:18:13 +03:00
2021-12-25 23:55:06 +02:00
const startWorker = async ( workerId ) => {
2021-05-02 00:19:18 +03:00
log . warn ( ` Starting worker ${ workerId } ` ) ;
2017-04-17 05:32:30 +03:00
const pgConfigs = {
development : {
2017-06-25 19:13:31 +03:00
user : process . env . DB _USER || pg . defaults . user ,
password : process . env . DB _PASS || pg . defaults . password ,
2017-10-17 12:45:37 +03:00
database : process . env . DB _NAME || 'mastodon_development' ,
2017-06-25 19:13:31 +03:00
host : process . env . DB _HOST || pg . defaults . host ,
port : process . env . DB _PORT || pg . defaults . port ,
2017-05-20 18:31:47 +03:00
max : 10 ,
2017-04-17 05:32:30 +03:00
} ,
production : {
user : process . env . DB _USER || 'mastodon' ,
password : process . env . DB _PASS || '' ,
database : process . env . DB _NAME || 'mastodon_production' ,
host : process . env . DB _HOST || 'localhost' ,
port : process . env . DB _PORT || 5432 ,
2017-05-20 18:31:47 +03:00
max : 10 ,
} ,
} ;
2017-02-02 02:31:09 +02:00
2019-03-11 01:51:23 +02:00
if ( ! ! process . env . DB _SSLMODE && process . env . DB _SSLMODE !== 'disable' ) {
pgConfigs . development . ssl = true ;
2021-12-25 23:55:06 +02:00
pgConfigs . production . ssl = true ;
2019-03-11 01:51:23 +02:00
}
const app = express ( ) ;
2020-08-11 19:24:59 +03:00
2022-04-19 10:11:58 +03:00
app . set ( 'trust proxy' , process . env . TRUSTED _PROXY _IP ? process . env . TRUSTED _PROXY _IP . split ( /(?:\s*,\s*|\s+)/ ) : 'loopback,uniquelocal' ) ;
2017-12-12 16:13:24 +02:00
2017-05-20 18:31:47 +03:00
const pgPool = new pg . Pool ( Object . assign ( pgConfigs [ env ] , dbUrlToConfig ( process . env . DATABASE _URL ) ) ) ;
const server = http . createServer ( app ) ;
const redisNamespace = process . env . REDIS _NAMESPACE || null ;
2017-02-07 15:37:12 +02:00
2017-05-07 20:42:32 +03:00
const redisParams = {
2021-12-25 23:55:06 +02:00
socket : {
host : process . env . REDIS _HOST || '127.0.0.1' ,
port : process . env . REDIS _PORT || 6379 ,
} ,
database : process . env . REDIS _DB || 0 ,
2020-06-24 23:25:23 +03:00
password : process . env . REDIS _PASSWORD || undefined ,
2017-05-20 18:31:47 +03:00
} ;
2017-05-07 20:42:32 +03:00
if ( redisNamespace ) {
2017-05-20 18:31:47 +03:00
redisParams . namespace = redisNamespace ;
2017-05-07 20:42:32 +03:00
}
2017-05-20 22:06:09 +03:00
2017-05-20 18:31:47 +03:00
const redisPrefix = redisNamespace ? ` ${ redisNamespace } : ` : '' ;
2017-05-07 20:42:32 +03:00
2022-03-21 20:08:29 +02:00
/ * *
* @ type { Object . < string , Array . < function ( string ) : void >> }
* /
const subs = { } ;
2021-12-25 23:55:06 +02:00
const redisSubscribeClient = await redisUrlToClient ( redisParams , process . env . REDIS _URL ) ;
const redisClient = await redisUrlToClient ( redisParams , process . env . REDIS _URL ) ;
2017-02-07 15:37:12 +02:00
2020-08-11 19:24:59 +03:00
/ * *
* @ param { string [ ] } channels
* @ return { function ( ) : void }
* /
2020-06-02 20:24:53 +03:00
const subscriptionHeartbeat = channels => {
const interval = 6 * 60 ;
2017-06-03 21:50:53 +03:00
const tellSubscribed = ( ) => {
2020-06-02 20:24:53 +03:00
channels . forEach ( channel => redisClient . set ( ` ${ redisPrefix } subscribed: ${ channel } ` , '1' , 'EX' , interval * 3 ) ) ;
2017-06-03 21:50:53 +03:00
} ;
2020-06-02 20:24:53 +03:00
2017-06-03 21:50:53 +03:00
tellSubscribed ( ) ;
2020-06-02 20:24:53 +03:00
const heartbeat = setInterval ( tellSubscribed , interval * 1000 ) ;
2017-06-03 21:50:53 +03:00
return ( ) => {
clearInterval ( heartbeat ) ;
} ;
} ;
2017-02-07 15:37:12 +02:00
2022-03-21 20:08:29 +02:00
/ * *
* @ param { string } message
* @ param { string } channel
* /
const onRedisMessage = ( message , channel ) => {
const callbacks = subs [ channel ] ;
log . silly ( ` New message on channel ${ channel } ` ) ;
if ( ! callbacks ) {
return ;
}
callbacks . forEach ( callback => callback ( message ) ) ;
} ;
2020-08-11 19:24:59 +03:00
/ * *
* @ param { string } channel
* @ param { function ( string ) : void } callback
* /
2017-04-17 05:32:30 +03:00
const subscribe = ( channel , callback ) => {
2017-05-20 18:31:47 +03:00
log . silly ( ` Adding listener for ${ channel } ` ) ;
2020-08-11 19:24:59 +03:00
2022-03-21 20:08:29 +02:00
subs [ channel ] = subs [ channel ] || [ ] ;
if ( subs [ channel ] . length === 0 ) {
log . verbose ( ` Subscribe ${ channel } ` ) ;
redisSubscribeClient . subscribe ( channel , onRedisMessage ) ;
}
subs [ channel ] . push ( callback ) ;
2017-05-20 18:31:47 +03:00
} ;
2017-02-03 19:27:42 +02:00
2020-08-11 19:24:59 +03:00
/ * *
* @ param { string } channel
* /
2022-01-07 20:50:12 +02:00
const unsubscribe = ( channel , callback ) => {
log . silly ( ` Removing listener for ${ channel } ` ) ;
2020-08-11 19:24:59 +03:00
2022-03-21 20:08:29 +02:00
if ( ! subs [ channel ] ) {
return ;
}
subs [ channel ] = subs [ channel ] . filter ( item => item !== callback ) ;
if ( subs [ channel ] . length === 0 ) {
log . verbose ( ` Unsubscribe ${ channel } ` ) ;
redisSubscribeClient . unsubscribe ( channel ) ;
delete subs [ channel ] ;
}
2017-05-20 18:31:47 +03:00
} ;
2017-02-03 19:27:42 +02:00
2020-08-11 19:24:59 +03:00
const FALSE _VALUES = [
false ,
0 ,
2020-11-23 18:35:14 +02:00
'0' ,
'f' ,
'F' ,
'false' ,
'FALSE' ,
'off' ,
'OFF' ,
2020-08-11 19:24:59 +03:00
] ;
/ * *
* @ param { any } value
* @ return { boolean }
* /
const isTruthy = value =>
value && ! FALSE _VALUES . includes ( value ) ;
/ * *
* @ param { any } req
* @ param { any } res
* @ param { function ( Error = ) : void }
* /
2017-04-17 05:32:30 +03:00
const allowCrossDomain = ( req , res , next ) => {
2017-05-20 18:31:47 +03:00
res . header ( 'Access-Control-Allow-Origin' , '*' ) ;
res . header ( 'Access-Control-Allow-Headers' , 'Authorization, Accept, Cache-Control' ) ;
res . header ( 'Access-Control-Allow-Methods' , 'GET, OPTIONS' ) ;
2017-02-06 00:37:25 +02:00
2017-05-20 18:31:47 +03:00
next ( ) ;
} ;
2017-02-06 00:37:25 +02:00
2020-08-11 19:24:59 +03:00
/ * *
* @ param { any } req
* @ param { any } res
* @ param { function ( Error = ) : void }
* /
2017-04-17 05:32:30 +03:00
const setRequestId = ( req , res , next ) => {
2017-05-20 18:31:47 +03:00
req . requestId = uuid . v4 ( ) ;
res . header ( 'X-Request-Id' , req . requestId ) ;
2017-02-02 02:31:09 +02:00
2017-05-20 18:31:47 +03:00
next ( ) ;
} ;
2017-02-02 02:31:09 +02:00
2020-08-11 19:24:59 +03:00
/ * *
* @ param { any } req
* @ param { any } res
* @ param { function ( Error = ) : void }
* /
2017-12-12 16:13:24 +02:00
const setRemoteAddress = ( req , res , next ) => {
req . remoteAddress = req . connection . remoteAddress ;
next ( ) ;
} ;
2021-09-26 14:23:28 +03:00
/ * *
* @ param { any } req
* @ param { string [ ] } necessaryScopes
* @ return { boolean }
* /
const isInScope = ( req , necessaryScopes ) =>
req . scopes . some ( scope => necessaryScopes . includes ( scope ) ) ;
2020-08-11 19:24:59 +03:00
/ * *
* @ param { string } token
* @ param { any } req
* @ return { Promise . < void > }
* /
const accountFromToken = ( token , req ) => new Promise ( ( resolve , reject ) => {
2017-04-17 05:32:30 +03:00
pgPool . connect ( ( err , client , done ) => {
2017-02-02 02:31:09 +02:00
if ( err ) {
2020-08-11 19:24:59 +03:00
reject ( err ) ;
2017-05-20 18:31:47 +03:00
return ;
2017-02-02 02:31:09 +02:00
}
2020-11-13 00:05:24 +02:00
client . query ( 'SELECT oauth_access_tokens.id, oauth_access_tokens.resource_owner_id, users.account_id, users.chosen_languages, oauth_access_tokens.scopes, devices.device_id FROM oauth_access_tokens INNER JOIN users ON oauth_access_tokens.resource_owner_id = users.id LEFT OUTER JOIN devices ON oauth_access_tokens.id = devices.access_token_id WHERE oauth_access_tokens.token = $1 AND oauth_access_tokens.revoked_at IS NULL LIMIT 1' , [ token ] , ( err , result ) => {
2017-05-20 18:31:47 +03:00
done ( ) ;
2017-02-02 02:31:09 +02:00
2017-04-17 05:32:30 +03:00
if ( err ) {
2020-08-11 19:24:59 +03:00
reject ( err ) ;
2017-05-20 18:31:47 +03:00
return ;
2017-04-17 05:32:30 +03:00
}
2017-02-02 02:31:09 +02:00
2017-04-17 05:32:30 +03:00
if ( result . rows . length === 0 ) {
2017-05-20 18:31:47 +03:00
err = new Error ( 'Invalid access token' ) ;
2020-08-11 19:24:59 +03:00
err . status = 401 ;
2019-05-24 16:21:42 +03:00
2020-08-11 19:24:59 +03:00
reject ( err ) ;
2019-05-24 16:21:42 +03:00
return ;
}
2020-11-13 00:05:24 +02:00
req . accessTokenId = result . rows [ 0 ] . id ;
2020-08-11 19:24:59 +03:00
req . scopes = result . rows [ 0 ] . scopes . split ( ' ' ) ;
2017-05-20 18:31:47 +03:00
req . accountId = result . rows [ 0 ] . account _id ;
2018-07-14 04:59:31 +03:00
req . chosenLanguages = result . rows [ 0 ] . chosen _languages ;
2020-06-02 20:24:53 +03:00
req . deviceId = result . rows [ 0 ] . device _id ;
2017-02-04 01:34:31 +02:00
2020-08-11 19:24:59 +03:00
resolve ( ) ;
2017-05-20 18:31:47 +03:00
} ) ;
} ) ;
2020-08-11 19:24:59 +03:00
} ) ;
2017-02-04 01:34:31 +02:00
2020-08-11 19:24:59 +03:00
/ * *
* @ param { any } req
* @ param { boolean = } required
* @ return { Promise . < void > }
* /
const accountFromRequest = ( req , required = true ) => new Promise ( ( resolve , reject ) => {
2017-05-29 19:20:53 +03:00
const authorization = req . headers . authorization ;
2020-08-11 19:24:59 +03:00
const location = url . parse ( req . url , true ) ;
const accessToken = location . query . access _token || req . headers [ 'sec-websocket-protocol' ] ;
2017-02-04 01:34:31 +02:00
2017-05-21 22:13:11 +03:00
if ( ! authorization && ! accessToken ) {
2017-12-12 16:13:24 +02:00
if ( required ) {
const err = new Error ( 'Missing access token' ) ;
2020-08-11 19:24:59 +03:00
err . status = 401 ;
2017-02-02 02:31:09 +02:00
2020-08-11 19:24:59 +03:00
reject ( err ) ;
2017-12-12 16:13:24 +02:00
return ;
} else {
2020-08-11 19:24:59 +03:00
resolve ( ) ;
2017-12-12 16:13:24 +02:00
return ;
}
2017-04-17 05:32:30 +03:00
}
2017-02-02 18:10:59 +02:00
2017-05-21 22:13:11 +03:00
const token = authorization ? authorization . replace ( /^Bearer / , '' ) : accessToken ;
2017-02-02 14:56:14 +02:00
2020-08-11 19:24:59 +03:00
resolve ( accountFromToken ( token , req ) ) ;
} ) ;
/ * *
* @ param { any } req
* @ return { string }
* /
const channelNameFromPath = req => {
const { path , query } = req ;
const onlyMedia = isTruthy ( query . only _media ) ;
2021-12-25 23:55:06 +02:00
switch ( path ) {
2020-08-11 19:24:59 +03:00
case '/api/v1/streaming/user' :
return 'user' ;
case '/api/v1/streaming/user/notification' :
return 'user:notification' ;
case '/api/v1/streaming/public' :
return onlyMedia ? 'public:media' : 'public' ;
case '/api/v1/streaming/public/local' :
return onlyMedia ? 'public:local:media' : 'public:local' ;
case '/api/v1/streaming/public/remote' :
return onlyMedia ? 'public:remote:media' : 'public:remote' ;
case '/api/v1/streaming/hashtag' :
return 'hashtag' ;
case '/api/v1/streaming/hashtag/local' :
return 'hashtag:local' ;
case '/api/v1/streaming/direct' :
return 'direct' ;
case '/api/v1/streaming/list' :
return 'list' ;
2020-11-23 18:35:14 +02:00
default :
return undefined ;
2020-08-11 19:24:59 +03:00
}
2017-05-20 18:31:47 +03:00
} ;
2017-02-05 04:19:04 +02:00
2020-08-11 19:24:59 +03:00
const PUBLIC _CHANNELS = [
2017-12-12 16:13:24 +02:00
'public' ,
2018-05-21 13:43:38 +03:00
'public:media' ,
2017-12-12 16:13:24 +02:00
'public:local' ,
2018-05-21 13:43:38 +03:00
'public:local:media' ,
2020-05-10 11:36:18 +03:00
'public:remote' ,
'public:remote:media' ,
2017-12-12 16:13:24 +02:00
'hashtag' ,
'hashtag:local' ,
] ;
2020-08-11 19:24:59 +03:00
/ * *
* @ param { any } req
* @ param { string } channelName
* @ return { Promise . < void > }
* /
const checkScopes = ( req , channelName ) => new Promise ( ( resolve , reject ) => {
log . silly ( req . requestId , ` Checking OAuth scopes for ${ channelName } ` ) ;
// When accessing public channels, no scopes are needed
if ( PUBLIC _CHANNELS . includes ( channelName ) ) {
resolve ( ) ;
return ;
}
2019-05-24 16:21:42 +03:00
2020-08-11 19:24:59 +03:00
// The `read` scope has the highest priority, if the token has it
// then it can access all streams
const requiredScopes = [ 'read' ] ;
// When accessing specifically the notifications stream,
// we need a read:notifications, while in all other cases,
// we can allow access with read:statuses. Mind that the
// user stream will not contain notifications unless
// the token has either read or read:notifications scope
// as well, this is handled separately.
if ( channelName === 'user:notification' ) {
requiredScopes . push ( 'read:notifications' ) ;
} else {
requiredScopes . push ( 'read:statuses' ) ;
2019-05-24 16:21:42 +03:00
}
2017-12-12 16:13:24 +02:00
2021-10-13 06:02:55 +03:00
if ( req . scopes && requiredScopes . some ( requiredScope => req . scopes . includes ( requiredScope ) ) ) {
2020-08-11 19:24:59 +03:00
resolve ( ) ;
return ;
}
2017-05-29 19:20:53 +03:00
2020-08-11 19:24:59 +03:00
const err = new Error ( 'Access token does not cover required scopes' ) ;
err . status = 401 ;
reject ( err ) ;
} ) ;
2017-12-12 16:13:24 +02:00
2020-08-11 19:24:59 +03:00
/ * *
* @ param { any } info
* @ param { function ( boolean , number , string ) : void } callback
* /
const wsVerifyClient = ( info , callback ) => {
// When verifying the websockets connection, we no longer pre-emptively
// check OAuth scopes and drop the connection if they're missing. We only
// drop the connection if access without token is not allowed by environment
// variables. OAuth scope checks are moved to the point of subscription
// to a specific stream.
accountFromRequest ( info . req , alwaysRequireAuth ) . then ( ( ) => {
callback ( true , undefined , undefined ) ;
} ) . catch ( err => {
log . error ( info . req . requestId , err . toString ( ) ) ;
callback ( false , 401 , 'Unauthorized' ) ;
} ) ;
} ;
2020-11-13 00:05:24 +02:00
/ * *
* @ typedef SystemMessageHandlers
* @ property { function ( ) : void } onKill
* /
/ * *
* @ param { any } req
* @ param { SystemMessageHandlers } eventHandlers
* @ return { function ( string ) : void }
* /
const createSystemMessageListener = ( req , eventHandlers ) => {
return message => {
2022-02-16 15:37:26 +02:00
const json = parseJSON ( message , req ) ;
2020-11-13 00:05:24 +02:00
if ( ! json ) return ;
const { event } = json ;
log . silly ( req . requestId , ` System message for ${ req . accountId } : ${ event } ` ) ;
if ( event === 'kill' ) {
log . verbose ( req . requestId , ` Closing connection for ${ req . accountId } due to expired access token ` ) ;
eventHandlers . onKill ( ) ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 10:42:13 +03:00
} else if ( event === 'filters_changed' ) {
log . verbose ( req . requestId , ` Invalidating filters cache for ${ req . accountId } ` ) ;
req . cachedFilters = null ;
2020-11-13 00:05:24 +02:00
}
2020-11-23 18:35:14 +02:00
} ;
2020-11-13 00:05:24 +02:00
} ;
/ * *
* @ param { any } req
* @ param { any } res
* /
const subscribeHttpToSystemChannel = ( req , res ) => {
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 10:42:13 +03:00
const accessTokenChannelId = ` timeline:access_token: ${ req . accessTokenId } ` ;
const systemChannelId = ` timeline:system: ${ req . accountId } ` ;
2020-11-13 00:05:24 +02:00
const listener = createSystemMessageListener ( req , {
2021-12-25 23:55:06 +02:00
onKill ( ) {
2020-11-13 00:05:24 +02:00
res . end ( ) ;
} ,
} ) ;
res . on ( 'close' , ( ) => {
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 10:42:13 +03:00
unsubscribe ( ` ${ redisPrefix } ${ accessTokenChannelId } ` , listener ) ;
2020-11-13 00:05:24 +02:00
unsubscribe ( ` ${ redisPrefix } ${ systemChannelId } ` , listener ) ;
} ) ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 10:42:13 +03:00
subscribe ( ` ${ redisPrefix } ${ accessTokenChannelId } ` , listener ) ;
2020-11-13 00:05:24 +02:00
subscribe ( ` ${ redisPrefix } ${ systemChannelId } ` , listener ) ;
} ;
2020-08-11 19:24:59 +03:00
/ * *
* @ param { any } req
* @ param { any } res
* @ param { function ( Error = ) : void } next
* /
2017-05-29 19:20:53 +03:00
const authenticationMiddleware = ( req , res , next ) => {
if ( req . method === 'OPTIONS' ) {
next ( ) ;
return ;
}
2020-08-11 19:24:59 +03:00
accountFromRequest ( req , alwaysRequireAuth ) . then ( ( ) => checkScopes ( req , channelNameFromPath ( req ) ) ) . then ( ( ) => {
2020-11-13 00:05:24 +02:00
subscribeHttpToSystemChannel ( req , res ) ;
} ) . then ( ( ) => {
2020-08-11 19:24:59 +03:00
next ( ) ;
} ) . catch ( err => {
next ( err ) ;
} ) ;
2017-05-29 19:20:53 +03:00
} ;
2020-08-11 19:24:59 +03:00
/ * *
* @ param { Error } err
* @ param { any } req
* @ param { any } res
* @ param { function ( Error = ) : void } next
* /
const errorMiddleware = ( err , req , res , next ) => {
2017-05-28 17:25:26 +03:00
log . error ( req . requestId , err . toString ( ) ) ;
2020-08-11 19:24:59 +03:00
if ( res . headersSent ) {
2020-11-23 18:35:14 +02:00
next ( err ) ;
return ;
2020-08-11 19:24:59 +03:00
}
res . writeHead ( err . status || 500 , { 'Content-Type' : 'application/json' } ) ;
res . end ( JSON . stringify ( { error : err . status ? err . toString ( ) : 'An unexpected error occurred' } ) ) ;
2017-05-20 18:31:47 +03:00
} ;
2017-02-05 04:19:04 +02:00
2020-08-11 19:24:59 +03:00
/ * *
2021-12-25 23:55:06 +02:00
* @ param { array } arr
2020-08-11 19:24:59 +03:00
* @ param { number = } shift
* @ return { string }
* /
2017-04-17 05:32:30 +03:00
const placeholders = ( arr , shift = 0 ) => arr . map ( ( _ , i ) => ` $ ${ i + 1 + shift } ` ) . join ( ', ' ) ;
2017-02-02 02:31:09 +02:00
2020-08-11 19:24:59 +03:00
/ * *
* @ param { string } listId
* @ param { any } req
* @ return { Promise . < void > }
* /
const authorizeListAccess = ( listId , req ) => new Promise ( ( resolve , reject ) => {
const { accountId } = req ;
2017-11-18 01:16:48 +02:00
pgPool . connect ( ( err , client , done ) => {
if ( err ) {
2020-08-11 19:24:59 +03:00
reject ( ) ;
2017-11-18 01:16:48 +02:00
return ;
}
2020-08-11 19:24:59 +03:00
client . query ( 'SELECT id, account_id FROM lists WHERE id = $1 LIMIT 1' , [ listId ] , ( err , result ) => {
2017-11-18 01:16:48 +02:00
done ( ) ;
2020-08-11 19:24:59 +03:00
if ( err || result . rows . length === 0 || result . rows [ 0 ] . account _id !== accountId ) {
reject ( ) ;
2017-11-18 01:16:48 +02:00
return ;
}
2020-08-11 19:24:59 +03:00
resolve ( ) ;
2017-11-18 01:16:48 +02:00
} ) ;
} ) ;
2020-08-11 19:24:59 +03:00
} ) ;
2017-11-18 01:16:48 +02:00
2020-08-11 19:24:59 +03:00
/ * *
* @ param { string [ ] } ids
* @ param { any } req
* @ param { function ( string , string ) : void } output
* @ param { function ( string [ ] , function ( string ) : void ) : void } attachCloseHandler
* @ param { boolean = } needsFiltering
* @ return { function ( string ) : void }
* /
2021-09-26 14:23:28 +03:00
const streamFrom = ( ids , req , output , attachCloseHandler , needsFiltering = false ) => {
2021-12-25 23:55:06 +02:00
const accountId = req . accountId || req . remoteAddress ;
2020-06-02 20:24:53 +03:00
2021-09-26 14:23:28 +03:00
log . verbose ( req . requestId , ` Starting stream from ${ ids . join ( ', ' ) } for ${ accountId } ` ) ;
2017-04-17 05:32:30 +03:00
const listener = message => {
2022-02-16 15:37:26 +02:00
const json = parseJSON ( message , req ) ;
2020-11-13 00:05:24 +02:00
2020-09-22 16:30:41 +03:00
if ( ! json ) return ;
2020-11-13 00:05:24 +02:00
2020-09-22 16:30:41 +03:00
const { event , payload , queued _at } = json ;
2017-02-02 14:56:14 +02:00
2017-04-17 05:32:30 +03:00
const transmit = ( ) => {
2021-12-25 23:55:06 +02:00
const now = new Date ( ) . getTime ( ) ;
const delta = now - queued _at ;
2017-09-24 16:31:03 +03:00
const encodedPayload = typeof payload === 'object' ? JSON . stringify ( payload ) : payload ;
2017-02-02 14:56:14 +02:00
2017-12-12 16:13:24 +02:00
log . silly ( req . requestId , ` Transmitting for ${ accountId } : ${ event } ${ encodedPayload } Delay: ${ delta } ms ` ) ;
2017-07-07 17:56:52 +03:00
output ( event , encodedPayload ) ;
2017-05-20 18:31:47 +03:00
} ;
2017-02-02 14:56:14 +02:00
2017-04-17 05:32:30 +03:00
// Only messages that may require filtering are statuses, since notifications
// are already personalized and deletes do not matter
2018-04-17 14:49:09 +03:00
if ( ! needsFiltering || event !== 'update' ) {
transmit ( ) ;
return ;
}
2017-02-02 14:56:14 +02:00
2021-12-25 23:55:06 +02:00
const unpackedPayload = payload ;
2018-04-17 14:49:09 +03:00
const targetAccountIds = [ unpackedPayload . account . id ] . concat ( unpackedPayload . mentions . map ( item => item . id ) ) ;
2021-12-25 23:55:06 +02:00
const accountDomain = unpackedPayload . account . acct . split ( '@' ) [ 1 ] ;
2017-04-17 05:32:30 +03:00
2018-07-14 04:59:31 +03:00
if ( Array . isArray ( req . chosenLanguages ) && unpackedPayload . language !== null && req . chosenLanguages . indexOf ( unpackedPayload . language ) === - 1 ) {
2018-04-17 14:49:09 +03:00
log . silly ( req . requestId , ` Message ${ unpackedPayload . id } filtered by language ( ${ unpackedPayload . language } ) ` ) ;
return ;
}
// When the account is not logged in, it is not necessary to confirm the block or mute
if ( ! req . accountId ) {
transmit ( ) ;
return ;
}
pgPool . connect ( ( err , client , done ) => {
if ( err ) {
log . error ( err ) ;
return ;
}
const queries = [
2021-12-25 23:55:06 +02:00
client . query ( ` SELECT 1
FROM blocks
WHERE ( account _id = $1 AND target _account _id IN ( $ { placeholders ( targetAccountIds , 2 ) } ) )
OR ( account _id = $2 AND target _account _id = $1 )
UNION
SELECT 1
FROM mutes
WHERE account _id = $1
AND target _account _id IN ( $ { placeholders ( targetAccountIds , 2 ) } ) ` , [req.accountId, unpackedPayload.account.id].concat(targetAccountIds)),
2018-04-17 14:49:09 +03:00
] ;
if ( accountDomain ) {
queries . push ( client . query ( 'SELECT 1 FROM account_domain_blocks WHERE account_id = $1 AND domain = $2' , [ req . accountId , accountDomain ] ) ) ;
}
2022-11-13 21:59:49 +02:00
if ( ! unpackedPayload . filtered && ! req . cachedFilters ) {
2022-11-15 03:09:58 +02:00
queries . push ( client . query ( 'SELECT filter.id AS id, filter.phrase AS title, filter.context AS context, filter.expires_at AS expires_at, filter.action AS filter_action, keyword.keyword AS keyword, keyword.whole_word AS whole_word FROM custom_filter_keywords keyword JOIN custom_filters filter ON keyword.custom_filter_id = filter.id WHERE filter.account_id = $1 AND (filter.expires_at IS NULL OR filter.expires_at > NOW())' , [ req . accountId ] ) ) ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 10:42:13 +03:00
}
2018-04-17 14:49:09 +03:00
Promise . all ( queries ) . then ( values => {
done ( ) ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 10:42:13 +03:00
if ( values [ 0 ] . rows . length > 0 || ( accountDomain && values [ 1 ] . rows . length > 0 ) ) {
2017-05-27 01:53:48 +03:00
return ;
}
2022-11-13 21:59:49 +02:00
if ( ! unpackedPayload . filtered && ! req . cachedFilters ) {
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 10:42:13 +03:00
const filterRows = values [ accountDomain ? 2 : 1 ] . rows ;
req . cachedFilters = filterRows . reduce ( ( cache , row ) => {
if ( cache [ row . id ] ) {
cache [ row . id ] . keywords . push ( [ row . keyword , row . whole _word ] ) ;
} else {
cache [ row . id ] = {
keywords : [ [ row . keyword , row . whole _word ] ] ,
expires _at : row . expires _at ,
repr : {
id : row . id ,
title : row . title ,
context : row . context ,
expires _at : row . expires _at ,
2022-11-13 21:59:49 +02:00
filter _action : [ 'warn' , 'hide' ] [ row . filter _action ] ,
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 10:42:13 +03:00
} ,
} ;
}
return cache ;
} , { } ) ;
Object . keys ( req . cachedFilters ) . forEach ( ( key ) => {
req . cachedFilters [ key ] . regexp = new RegExp ( req . cachedFilters [ key ] . keywords . map ( ( [ keyword , whole _word ] ) => {
2022-12-18 17:51:37 +02:00
let expr = keyword . replace ( /[.*+?^${}()|[\]\\]/g , '\\$&' ) ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 10:42:13 +03:00
if ( whole _word ) {
if ( /^[\w]/ . test ( expr ) ) {
expr = ` \\ b ${ expr } ` ;
}
if ( /[\w]$/ . test ( expr ) ) {
expr = ` ${ expr } \\ b ` ;
}
}
return expr ;
} ) . join ( '|' ) , 'i' ) ;
} ) ;
}
// Check filters
2022-11-13 21:59:49 +02:00
if ( req . cachedFilters && ! unpackedPayload . filtered ) {
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 10:42:13 +03:00
const status = unpackedPayload ;
const searchContent = ( [ status . spoiler _text || '' , status . content ] . concat ( ( status . poll && status . poll . options ) ? status . poll . options . map ( option => option . title ) : [ ] ) ) . concat ( status . media _attachments . map ( att => att . description ) ) . join ( '\n\n' ) . replace ( /<br\s*\/?>/g , '\n' ) . replace ( /<\/p><p>/g , '\n\n' ) ;
const searchIndex = JSDOM . fragment ( searchContent ) . textContent ;
const now = new Date ( ) ;
2022-11-13 21:59:49 +02:00
payload . filtered = [ ] ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 10:42:13 +03:00
Object . values ( req . cachedFilters ) . forEach ( ( cachedFilter ) => {
if ( ( cachedFilter . expires _at === null || cachedFilter . expires _at > now ) ) {
const keyword _matches = searchIndex . match ( cachedFilter . regexp ) ;
if ( keyword _matches ) {
2022-11-13 21:59:49 +02:00
payload . filtered . push ( {
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 10:42:13 +03:00
filter : cachedFilter . repr ,
keyword _matches ,
} ) ;
}
}
} ) ;
}
2018-04-17 14:49:09 +03:00
transmit ( ) ;
} ) . catch ( err => {
log . error ( err ) ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 10:42:13 +03:00
done ( ) ;
2017-05-20 18:31:47 +03:00
} ) ;
2018-04-17 14:49:09 +03:00
} ) ;
2017-05-20 18:31:47 +03:00
} ;
2017-04-17 05:32:30 +03:00
2020-06-02 20:24:53 +03:00
ids . forEach ( id => {
subscribe ( ` ${ redisPrefix } ${ id } ` , listener ) ;
} ) ;
2020-08-11 19:24:59 +03:00
if ( attachCloseHandler ) {
attachCloseHandler ( ids . map ( id => ` ${ redisPrefix } ${ id } ` ) , listener ) ;
}
return listener ;
2017-05-20 18:31:47 +03:00
} ;
2017-02-02 02:31:09 +02:00
2020-08-11 19:24:59 +03:00
/ * *
* @ param { any } req
* @ param { any } res
* @ return { function ( string , string ) : void }
* /
2017-04-17 05:32:30 +03:00
const streamToHttp = ( req , res ) => {
2017-12-12 16:13:24 +02:00
const accountId = req . accountId || req . remoteAddress ;
2017-05-20 18:31:47 +03:00
res . setHeader ( 'Content-Type' , 'text/event-stream' ) ;
2020-01-24 21:51:33 +02:00
res . setHeader ( 'Cache-Control' , 'no-store' ) ;
2017-05-20 18:31:47 +03:00
res . setHeader ( 'Transfer-Encoding' , 'chunked' ) ;
2017-02-04 01:34:31 +02:00
2020-01-24 21:51:33 +02:00
res . write ( ':)\n' ) ;
2017-05-20 18:31:47 +03:00
const heartbeat = setInterval ( ( ) => res . write ( ':thump\n' ) , 15000 ) ;
2017-02-04 01:34:31 +02:00
2017-04-17 05:32:30 +03:00
req . on ( 'close' , ( ) => {
2017-12-12 16:13:24 +02:00
log . verbose ( req . requestId , ` Ending stream for ${ accountId } ` ) ;
2017-05-20 18:31:47 +03:00
clearInterval ( heartbeat ) ;
} ) ;
2017-02-02 16:20:31 +02:00
2017-04-17 05:32:30 +03:00
return ( event , payload ) => {
2017-05-20 18:31:47 +03:00
res . write ( ` event: ${ event } \n ` ) ;
res . write ( ` data: ${ payload } \n \n ` ) ;
} ;
} ;
2017-02-02 02:31:09 +02:00
2020-08-11 19:24:59 +03:00
/ * *
* @ param { any } req
* @ param { function ( ) : void } [ closeHandler ]
2021-12-25 23:55:06 +02:00
* @ return { function ( string [ ] ) : void }
2020-08-11 19:24:59 +03:00
* /
2021-12-25 23:55:06 +02:00
const streamHttpEnd = ( req , closeHandler = undefined ) => ( ids ) => {
2017-04-17 05:32:30 +03:00
req . on ( 'close' , ( ) => {
2020-06-02 20:24:53 +03:00
ids . forEach ( id => {
2021-12-25 23:55:06 +02:00
unsubscribe ( id ) ;
2020-06-02 20:24:53 +03:00
} ) ;
2017-06-03 21:50:53 +03:00
if ( closeHandler ) {
closeHandler ( ) ;
}
2017-05-20 18:31:47 +03:00
} ) ;
} ;
2017-02-04 01:34:31 +02:00
2020-08-11 19:24:59 +03:00
/ * *
* @ param { any } req
* @ param { any } ws
* @ param { string [ ] } streamName
* @ return { function ( string , string ) : void }
* /
const streamToWs = ( req , ws , streamName ) => ( event , payload ) => {
2017-05-28 17:25:26 +03:00
if ( ws . readyState !== ws . OPEN ) {
log . error ( req . requestId , 'Tried writing to closed socket' ) ;
return ;
}
2017-02-04 01:34:31 +02:00
2020-08-11 19:24:59 +03:00
ws . send ( JSON . stringify ( { stream : streamName , event , payload } ) ) ;
2017-05-20 18:31:47 +03:00
} ;
2017-02-04 01:34:31 +02:00
2020-08-11 19:24:59 +03:00
/ * *
* @ param { any } res
* /
2018-10-11 20:24:43 +03:00
const httpNotFound = res => {
res . writeHead ( 404 , { 'Content-Type' : 'application/json' } ) ;
res . end ( JSON . stringify ( { error : 'Not found' } ) ) ;
} ;
2017-05-20 18:31:47 +03:00
app . use ( setRequestId ) ;
2017-12-12 16:13:24 +02:00
app . use ( setRemoteAddress ) ;
2017-05-20 18:31:47 +03:00
app . use ( allowCrossDomain ) ;
2018-08-26 12:54:25 +03:00
app . get ( '/api/v1/streaming/health' , ( req , res ) => {
res . writeHead ( 200 , { 'Content-Type' : 'text/plain' } ) ;
res . end ( 'OK' ) ;
} ) ;
2023-02-03 22:29:23 +02:00
app . get ( '/metrics' , ( req , res ) => server . getConnections ( ( err , count ) => {
res . writeHeader ( 200 , { 'Content-Type' : 'application/openmetrics-text; version=1.0.0; charset=utf-8' } ) ;
res . write ( '# TYPE connected_clients gauge\n' ) ;
res . write ( '# HELP connected_clients The number of clients connected to the streaming server\n' ) ;
res . write ( ` connected_clients ${ count } .0 \n ` ) ;
res . write ( '# TYPE connected_channels gauge\n' ) ;
res . write ( '# HELP connected_channels The number of Redis channels the streaming server is subscribed to\n' ) ;
res . write ( ` connected_channels ${ Object . keys ( subs ) . length } .0 \n ` ) ;
res . write ( '# EOF\n' ) ;
res . end ( ) ;
} ) ) ;
2017-05-20 18:31:47 +03:00
app . use ( authenticationMiddleware ) ;
app . use ( errorMiddleware ) ;
2017-02-02 02:31:09 +02:00
2020-08-11 19:24:59 +03:00
app . get ( '/api/v1/streaming/*' , ( req , res ) => {
channelNameToIds ( req , channelNameFromPath ( req ) , req . query ) . then ( ( { channelIds , options } ) => {
const onSend = streamToHttp ( req , res ) ;
2021-12-25 23:55:06 +02:00
const onEnd = streamHttpEnd ( req , subscriptionHeartbeat ( channelIds ) ) ;
2018-05-21 13:43:38 +03:00
2021-09-26 14:23:28 +03:00
streamFrom ( channelIds , req , onSend , onEnd , options . needsFiltering ) ;
2020-08-11 19:24:59 +03:00
} ) . catch ( err => {
log . verbose ( req . requestId , 'Subscription error:' , err . toString ( ) ) ;
2018-10-11 20:24:43 +03:00
httpNotFound ( res ) ;
2017-11-18 01:16:48 +02:00
} ) ;
} ) ;
2021-03-24 10:37:41 +02:00
const wss = new WebSocket . Server ( { server , verifyClient : wsVerifyClient } ) ;
2017-05-29 19:20:53 +03:00
2020-08-11 19:24:59 +03:00
/ * *
* @ typedef StreamParams
* @ property { string } [ tag ]
* @ property { string } [ list ]
* @ property { string } [ only _media ]
* /
2021-09-26 14:23:28 +03:00
/ * *
* @ param { any } req
* @ return { string [ ] }
* /
const channelsForUserStream = req => {
const arr = [ ` timeline: ${ req . accountId } ` ] ;
if ( isInScope ( req , [ 'crypto' ] ) && req . deviceId ) {
arr . push ( ` timeline: ${ req . accountId } : ${ req . deviceId } ` ) ;
}
if ( isInScope ( req , [ 'read' , 'read:notifications' ] ) ) {
arr . push ( ` timeline: ${ req . accountId } :notifications ` ) ;
}
return arr ;
} ;
2022-07-13 16:03:28 +03:00
/ * *
* See app / lib / ascii _folder . rb for the canon definitions
* of these constants
* /
const NON _ASCII _CHARS = 'ÀÁÂÃÄÅàáâãäåĀāĂ㥹ÇçĆćĈĉĊċČčÐðĎďĐđÈÉÊËèéêëĒēĔĕĖėĘęĚěĜĝĞğĠġĢģĤĥĦħÌÍÎÏìíîïĨĩĪīĬĭĮįİıĴĵĶķĸĹĺĻļĽľĿŀŁłÑñŃńŅņŇňʼnŊŋÒÓÔÕÖØòóôõöøŌōŎŏŐőŔŕŖŗŘřŚśŜŝŞşŠšſŢţŤťŦŧÙÚÛÜùúûüŨũŪūŬŭŮůŰűŲųŴŵÝýÿŶŷŸŹźŻżŽž' ;
const EQUIVALENT _ASCII _CHARS = 'AAAAAAaaaaaaAaAaAaCcCcCcCcCcDdDdDdEEEEeeeeEeEeEeEeEeGgGgGgGgHhHhIIIIiiiiIiIiIiIiIiJjKkkLlLlLlLlLlNnNnNnNnnNnOOOOOOooooooOoOoOoRrRrRrSsSsSsSssTtTtTtUUUUuuuuUuUuUuUuUuUuWwYyyYyYZzZzZz' ;
/ * *
* @ param { string } str
* @ return { string }
* /
const foldToASCII = str => {
const regex = new RegExp ( NON _ASCII _CHARS . split ( '' ) . join ( '|' ) , 'g' ) ;
return str . replace ( regex , match => {
const index = NON _ASCII _CHARS . indexOf ( match ) ;
return EQUIVALENT _ASCII _CHARS [ index ] ;
} ) ;
} ;
/ * *
* @ param { string } str
* @ return { string }
* /
const normalizeHashtag = str => {
return foldToASCII ( str . normalize ( 'NFKC' ) . toLowerCase ( ) ) . replace ( /[^\p{L}\p{N}_\u00b7\u200c]/gu , '' ) ;
} ;
2020-08-11 19:24:59 +03:00
/ * *
* @ param { any } req
* @ param { string } name
* @ param { StreamParams } params
2021-09-26 14:23:28 +03:00
* @ return { Promise . < { channelIds : string [ ] , options : { needsFiltering : boolean } } > }
2020-08-11 19:24:59 +03:00
* /
const channelNameToIds = ( req , name , params ) => new Promise ( ( resolve , reject ) => {
2021-12-25 23:55:06 +02:00
switch ( name ) {
2017-05-29 19:20:53 +03:00
case 'user' :
2020-08-11 19:24:59 +03:00
resolve ( {
2021-09-26 14:23:28 +03:00
channelIds : channelsForUserStream ( req ) ,
options : { needsFiltering : false } ,
2020-08-11 19:24:59 +03:00
} ) ;
2020-06-02 20:24:53 +03:00
2017-06-03 21:50:53 +03:00
break ;
case 'user:notification' :
2020-08-11 19:24:59 +03:00
resolve ( {
2021-09-26 14:23:28 +03:00
channelIds : [ ` timeline: ${ req . accountId } :notifications ` ] ,
options : { needsFiltering : false } ,
2020-08-11 19:24:59 +03:00
} ) ;
2017-05-29 19:20:53 +03:00
break ;
case 'public' :
2020-08-11 19:24:59 +03:00
resolve ( {
channelIds : [ 'timeline:public' ] ,
2021-09-26 14:23:28 +03:00
options : { needsFiltering : true } ,
2020-08-11 19:24:59 +03:00
} ) ;
2017-05-29 19:20:53 +03:00
break ;
case 'public:local' :
2020-08-11 19:24:59 +03:00
resolve ( {
channelIds : [ 'timeline:public:local' ] ,
2021-09-26 14:23:28 +03:00
options : { needsFiltering : true } ,
2020-08-11 19:24:59 +03:00
} ) ;
2017-05-29 19:20:53 +03:00
break ;
2020-05-10 11:36:18 +03:00
case 'public:remote' :
2020-08-11 19:24:59 +03:00
resolve ( {
channelIds : [ 'timeline:public:remote' ] ,
2021-09-26 14:23:28 +03:00
options : { needsFiltering : true } ,
2020-08-11 19:24:59 +03:00
} ) ;
2020-05-10 11:36:18 +03:00
break ;
2018-05-21 13:43:38 +03:00
case 'public:media' :
2020-08-11 19:24:59 +03:00
resolve ( {
channelIds : [ 'timeline:public:media' ] ,
2021-09-26 14:23:28 +03:00
options : { needsFiltering : true } ,
2020-08-11 19:24:59 +03:00
} ) ;
2018-05-21 13:43:38 +03:00
break ;
case 'public:local:media' :
2020-08-11 19:24:59 +03:00
resolve ( {
channelIds : [ 'timeline:public:local:media' ] ,
2021-09-26 14:23:28 +03:00
options : { needsFiltering : true } ,
2020-08-11 19:24:59 +03:00
} ) ;
2018-05-21 13:43:38 +03:00
break ;
2020-05-10 11:36:18 +03:00
case 'public:remote:media' :
2020-08-11 19:24:59 +03:00
resolve ( {
channelIds : [ 'timeline:public:remote:media' ] ,
2021-09-26 14:23:28 +03:00
options : { needsFiltering : true } ,
2020-08-11 19:24:59 +03:00
} ) ;
2020-05-10 11:36:18 +03:00
break ;
2018-04-18 14:09:06 +03:00
case 'direct' :
2020-08-11 19:24:59 +03:00
resolve ( {
channelIds : [ ` timeline:direct: ${ req . accountId } ` ] ,
2021-09-26 14:23:28 +03:00
options : { needsFiltering : false } ,
2020-08-11 19:24:59 +03:00
} ) ;
2018-04-18 14:09:06 +03:00
break ;
2017-05-29 19:20:53 +03:00
case 'hashtag' :
2020-08-11 19:24:59 +03:00
if ( ! params . tag || params . tag . length === 0 ) {
reject ( 'No tag for stream provided' ) ;
} else {
resolve ( {
2022-07-13 16:03:28 +03:00
channelIds : [ ` timeline:hashtag: ${ normalizeHashtag ( params . tag ) } ` ] ,
2021-09-26 14:23:28 +03:00
options : { needsFiltering : true } ,
2020-08-11 19:24:59 +03:00
} ) ;
2018-10-11 20:24:43 +03:00
}
2017-05-29 19:20:53 +03:00
break ;
case 'hashtag:local' :
2020-08-11 19:24:59 +03:00
if ( ! params . tag || params . tag . length === 0 ) {
reject ( 'No tag for stream provided' ) ;
} else {
resolve ( {
2022-07-13 16:03:28 +03:00
channelIds : [ ` timeline:hashtag: ${ normalizeHashtag ( params . tag ) } :local ` ] ,
2021-09-26 14:23:28 +03:00
options : { needsFiltering : true } ,
2020-08-11 19:24:59 +03:00
} ) ;
2018-10-11 20:24:43 +03:00
}
2017-05-29 19:20:53 +03:00
break ;
2017-11-18 01:16:48 +02:00
case 'list' :
2020-08-11 19:24:59 +03:00
authorizeListAccess ( params . list , req ) . then ( ( ) => {
resolve ( {
channelIds : [ ` timeline:list: ${ params . list } ` ] ,
2021-09-26 14:23:28 +03:00
options : { needsFiltering : false } ,
2020-08-11 19:24:59 +03:00
} ) ;
} ) . catch ( ( ) => {
reject ( 'Not authorized to stream this list' ) ;
2017-11-18 01:16:48 +02:00
} ) ;
2020-08-11 19:24:59 +03:00
2017-11-18 01:16:48 +02:00
break ;
2017-05-29 19:20:53 +03:00
default :
2020-08-11 19:24:59 +03:00
reject ( 'Unknown stream type' ) ;
}
} ) ;
/ * *
* @ param { string } channelName
* @ param { StreamParams } params
* @ return { string [ ] }
* /
const streamNameFromChannelName = ( channelName , params ) => {
if ( channelName === 'list' ) {
return [ channelName , params . list ] ;
} else if ( [ 'hashtag' , 'hashtag:local' ] . includes ( channelName ) ) {
return [ channelName , params . tag ] ;
} else {
return [ channelName ] ;
}
} ;
/ * *
* @ typedef WebSocketSession
* @ property { any } socket
* @ property { any } request
* @ property { Object . < string , { listener : function ( string ) : void , stopHeartbeat : function ( ) : void } > } subscriptions
* /
/ * *
* @ param { WebSocketSession } session
* @ param { string } channelName
* @ param { StreamParams } params
* /
const subscribeWebsocketToChannel = ( { socket , request , subscriptions } , channelName , params ) =>
2021-12-25 23:55:06 +02:00
checkScopes ( request , channelName ) . then ( ( ) => channelNameToIds ( request , channelName , params ) ) . then ( ( {
channelIds ,
options ,
} ) => {
2020-08-11 19:24:59 +03:00
if ( subscriptions [ channelIds . join ( ';' ) ] ) {
return ;
}
2021-12-25 23:55:06 +02:00
const onSend = streamToWs ( request , socket , streamNameFromChannelName ( channelName , params ) ) ;
2020-08-11 19:24:59 +03:00
const stopHeartbeat = subscriptionHeartbeat ( channelIds ) ;
2021-12-25 23:55:06 +02:00
const listener = streamFrom ( channelIds , request , onSend , undefined , options . needsFiltering ) ;
2020-08-11 19:24:59 +03:00
subscriptions [ channelIds . join ( ';' ) ] = {
listener ,
stopHeartbeat ,
} ;
} ) . catch ( err => {
log . verbose ( request . requestId , 'Subscription error:' , err . toString ( ) ) ;
socket . send ( JSON . stringify ( { error : err . toString ( ) } ) ) ;
} ) ;
/ * *
* @ param { WebSocketSession } session
* @ param { string } channelName
* @ param { StreamParams } params
* /
const unsubscribeWebsocketFromChannel = ( { socket , request , subscriptions } , channelName , params ) =>
channelNameToIds ( request , channelName , params ) . then ( ( { channelIds } ) => {
log . verbose ( request . requestId , ` Ending stream from ${ channelIds . join ( ', ' ) } for ${ request . accountId } ` ) ;
2020-08-12 16:36:07 +03:00
const subscription = subscriptions [ channelIds . join ( ';' ) ] ;
2020-08-11 19:24:59 +03:00
2020-08-12 16:36:07 +03:00
if ( ! subscription ) {
2020-08-11 19:24:59 +03:00
return ;
}
2020-08-12 16:36:07 +03:00
const { listener , stopHeartbeat } = subscription ;
2020-08-11 19:24:59 +03:00
channelIds . forEach ( channelId => {
unsubscribe ( ` ${ redisPrefix } ${ channelId } ` , listener ) ;
} ) ;
stopHeartbeat ( ) ;
2020-08-12 16:36:07 +03:00
delete subscriptions [ channelIds . join ( ';' ) ] ;
2020-08-11 19:24:59 +03:00
} ) . catch ( err => {
log . verbose ( request . requestId , 'Unsubscription error:' , err ) ;
socket . send ( JSON . stringify ( { error : err . toString ( ) } ) ) ;
} ) ;
2020-11-13 00:05:24 +02:00
/ * *
* @ param { WebSocketSession } session
* /
const subscribeWebsocketToSystemChannel = ( { socket , request , subscriptions } ) => {
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 10:42:13 +03:00
const accessTokenChannelId = ` timeline:access_token: ${ request . accessTokenId } ` ;
const systemChannelId = ` timeline:system: ${ request . accountId } ` ;
2020-11-13 00:05:24 +02:00
const listener = createSystemMessageListener ( request , {
2021-12-25 23:55:06 +02:00
onKill ( ) {
2020-11-13 00:05:24 +02:00
socket . close ( ) ;
} ,
} ) ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 10:42:13 +03:00
subscribe ( ` ${ redisPrefix } ${ accessTokenChannelId } ` , listener ) ;
2020-11-13 00:05:24 +02:00
subscribe ( ` ${ redisPrefix } ${ systemChannelId } ` , listener ) ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 10:42:13 +03:00
subscriptions [ accessTokenChannelId ] = {
listener ,
stopHeartbeat : ( ) => {
} ,
} ;
2020-11-13 00:05:24 +02:00
subscriptions [ systemChannelId ] = {
listener ,
2021-12-25 23:55:06 +02:00
stopHeartbeat : ( ) => {
} ,
2020-11-13 00:05:24 +02:00
} ;
} ;
2020-08-11 19:24:59 +03:00
/ * *
* @ param { string | string [ ] } arrayOrString
* @ return { string }
* /
const firstParam = arrayOrString => {
if ( Array . isArray ( arrayOrString ) ) {
return arrayOrString [ 0 ] ;
} else {
return arrayOrString ;
}
} ;
wss . on ( 'connection' , ( ws , req ) => {
const location = url . parse ( req . url , true ) ;
2021-12-25 23:55:06 +02:00
req . requestId = uuid . v4 ( ) ;
2020-08-11 19:24:59 +03:00
req . remoteAddress = ws . _socket . remoteAddress ;
2021-03-24 10:37:41 +02:00
ws . isAlive = true ;
ws . on ( 'pong' , ( ) => {
ws . isAlive = true ;
} ) ;
2020-08-11 19:24:59 +03:00
/ * *
* @ type { WebSocketSession }
* /
const session = {
socket : ws ,
request : req ,
subscriptions : { } ,
} ;
const onEnd = ( ) => {
const keys = Object . keys ( session . subscriptions ) ;
keys . forEach ( channelIds => {
const { listener , stopHeartbeat } = session . subscriptions [ channelIds ] ;
channelIds . split ( ';' ) . forEach ( channelId => {
unsubscribe ( ` ${ redisPrefix } ${ channelId } ` , listener ) ;
} ) ;
stopHeartbeat ( ) ;
} ) ;
} ;
ws . on ( 'close' , onEnd ) ;
ws . on ( 'error' , onEnd ) ;
ws . on ( 'message' , data => {
2022-02-16 15:37:26 +02:00
const json = parseJSON ( data , session . request ) ;
2020-11-13 00:05:24 +02:00
2020-09-22 16:30:41 +03:00
if ( ! json ) return ;
2020-11-13 00:05:24 +02:00
2020-09-22 16:30:41 +03:00
const { type , stream , ... params } = json ;
2020-08-11 19:24:59 +03:00
if ( type === 'subscribe' ) {
subscribeWebsocketToChannel ( session , firstParam ( stream ) , params ) ;
} else if ( type === 'unsubscribe' ) {
2020-11-23 18:35:14 +02:00
unsubscribeWebsocketFromChannel ( session , firstParam ( stream ) , params ) ;
2020-08-11 19:24:59 +03:00
} else {
// Unknown action type
}
} ) ;
2020-11-13 00:05:24 +02:00
subscribeWebsocketToSystemChannel ( session ) ;
2020-08-11 19:24:59 +03:00
if ( location . query . stream ) {
subscribeWebsocketToChannel ( session , firstParam ( location . query . stream ) , location . query ) ;
2017-05-29 19:20:53 +03:00
}
2017-05-20 18:31:47 +03:00
} ) ;
2017-02-04 01:34:31 +02:00
2021-03-24 10:37:41 +02:00
setInterval ( ( ) => {
wss . clients . forEach ( ws => {
if ( ws . isAlive === false ) {
ws . terminate ( ) ;
return ;
}
ws . isAlive = false ;
2021-05-02 15:30:26 +03:00
ws . ping ( '' , false ) ;
2021-03-24 10:37:41 +02:00
} ) ;
} , 30000 ) ;
2017-05-28 17:25:26 +03:00
2018-10-20 03:25:25 +03:00
attachServerWithConfig ( server , address => {
2021-05-02 00:19:18 +03:00
log . warn ( ` Worker ${ workerId } now listening on ${ address } ` ) ;
2018-10-20 03:25:25 +03:00
} ) ;
2017-04-21 20:24:31 +03:00
2017-05-28 17:25:26 +03:00
const onExit = ( ) => {
2021-05-02 00:19:18 +03:00
log . warn ( ` Worker ${ workerId } exiting ` ) ;
2017-05-20 18:31:47 +03:00
server . close ( ) ;
2017-07-07 21:01:00 +03:00
process . exit ( 0 ) ;
2017-05-28 17:25:26 +03:00
} ;
const onError = ( err ) => {
log . error ( err ) ;
2017-12-12 21:19:33 +02:00
server . close ( ) ;
process . exit ( 0 ) ;
2017-05-28 17:25:26 +03:00
} ;
process . on ( 'SIGINT' , onExit ) ;
process . on ( 'SIGTERM' , onExit ) ;
process . on ( 'exit' , onExit ) ;
2017-12-12 21:19:33 +02:00
process . on ( 'uncaughtException' , onError ) ;
2017-05-28 17:25:26 +03:00
} ;
2020-08-11 19:24:59 +03:00
/ * *
* @ param { any } server
* @ param { function ( string ) : void } [ onSuccess ]
* /
2018-10-20 03:25:25 +03:00
const attachServerWithConfig = ( server , onSuccess ) => {
if ( process . env . SOCKET || process . env . PORT && isNaN ( + process . env . PORT ) ) {
server . listen ( process . env . SOCKET || process . env . PORT , ( ) => {
if ( onSuccess ) {
2018-10-21 17:41:33 +03:00
fs . chmodSync ( server . address ( ) , 0o666 ) ;
2018-10-20 03:25:25 +03:00
onSuccess ( server . address ( ) ) ;
}
} ) ;
} else {
2019-07-15 06:56:35 +03:00
server . listen ( + process . env . PORT || 4000 , process . env . BIND || '127.0.0.1' , ( ) => {
2018-10-20 03:25:25 +03:00
if ( onSuccess ) {
onSuccess ( ` ${ server . address ( ) . address } : ${ server . address ( ) . port } ` ) ;
}
} ) ;
}
} ;
2020-08-11 19:24:59 +03:00
/ * *
* @ param { function ( Error = ) : void } onSuccess
* /
2018-10-20 03:25:25 +03:00
const onPortAvailable = onSuccess => {
const testServer = http . createServer ( ) ;
testServer . once ( 'error' , err => {
onSuccess ( err ) ;
} ) ;
testServer . once ( 'listening' , ( ) => {
testServer . once ( 'close' , ( ) => onSuccess ( ) ) ;
testServer . close ( ) ;
} ) ;
attachServerWithConfig ( testServer ) ;
} ;
onPortAvailable ( err => {
if ( err ) {
log . error ( 'Could not start server, the port or socket is in use' ) ;
return ;
}
throng ( {
workers : numWorkers ,
lifetime : Infinity ,
start : startWorker ,
master : startMaster ,
} ) ;
2017-05-28 17:25:26 +03:00
} ) ;