Allow keywords to match either substrings or whole words.
Word-boundary matching only works as intended in English and languages
that use similar word-breaking characters; it doesn't work so well in
(say) Japanese, Chinese, or Thai.  It's unacceptable to have a feature
that doesn't work as intended for some languages.  (Moreso especially
considering that it's likely that the largest contingent on the Mastodon
bit of the fediverse speaks Japanese.)
There are rules specified in Unicode TR29[1] for word-breaking across
all languages supported by Unicode, but the rules deliberately do not
cover all cases.  In fact, TR29 states
    For example, reliable detection of word boundaries in languages such
    as Thai, Lao, Chinese, or Japanese requires the use of dictionary
    lookup, analogous to English hyphenation.
So we aren't going to be able to make word detection work with regexes
within Mastodon (or glitchsoc).  However, for a first pass (even if it's
kind of punting) we can allow the user to choose whether they want word
or substring detection and warn about the limitations of this
implementation in, say, docs.
[1]: https://unicode.org/reports/tr29/
     https://web.archive.org/web/20171001005125/https://unicode.org/reports/tr29/
			
			
This commit is contained in:
		
							parent
							
								
									018657a0e0
								
							
						
					
					
						commit
						05ee0aeb8b
					
				
					 4 changed files with 16 additions and 6 deletions
				
			
		|  | @ -6,6 +6,7 @@ | |||
| #  id         :integer          not null, primary key | ||||
| #  account_id :integer          not null | ||||
| #  keyword    :string           not null | ||||
| #  whole_word :boolean          default(TRUE), not null | ||||
| #  created_at :datetime         not null | ||||
| #  updated_at :datetime         not null | ||||
| # | ||||
|  | @ -32,12 +33,13 @@ class KeywordMute < ApplicationRecord | |||
| 
 | ||||
|     def initialize(account_id) | ||||
|       re = [].tap do |arr| | ||||
|         KeywordMute.where(account_id: account_id).select(:keyword, :id).find_each do |m| | ||||
|           arr << Regexp.escape(m.keyword.strip) | ||||
|         KeywordMute.where(account_id: account_id).select(:keyword, :id, :whole_word).find_each do |m| | ||||
|           boundary = m.whole_word ? '\b' : '' | ||||
|           arr << "#{boundary}#{Regexp.escape(m.keyword.strip)}#{boundary}" | ||||
|         end | ||||
|       end.join('|') | ||||
| 
 | ||||
|       @regex = /\b(?:#{re})\b/i unless re.empty? | ||||
|       @regex = /#{re}/i unless re.empty? | ||||
|     end | ||||
| 
 | ||||
|     def =~(str) | ||||
|  |  | |||
|  | @ -3,6 +3,7 @@ class CreateKeywordMutes < ActiveRecord::Migration[5.1] | |||
|     create_table :keyword_mutes do |t| | ||||
|       t.references :account, null: false | ||||
|       t.string :keyword, null: false | ||||
|       t.boolean :whole_word, null: false, default: true | ||||
|       t.timestamps | ||||
|     end | ||||
| 
 | ||||
|  |  | |||
|  | @ -170,6 +170,7 @@ ActiveRecord::Schema.define(version: 20171010025614) do | |||
|   create_table "keyword_mutes", force: :cascade do |t| | ||||
|     t.bigint "account_id", null: false | ||||
|     t.string "keyword", null: false | ||||
|     t.boolean "whole_word", default: true, null: false | ||||
|     t.datetime "created_at", null: false | ||||
|     t.datetime "updated_at", null: false | ||||
|     t.index ["account_id"], name: "index_keyword_mutes_on_account_id" | ||||
|  |  | |||
|  | @ -30,10 +30,16 @@ RSpec.describe KeywordMute, type: :model do | |||
|         expect(matcher =~ 'This is a hot take').to be_falsy | ||||
|       end | ||||
| 
 | ||||
|       it 'does not match substrings matching keywords' do | ||||
|         KeywordMute.create!(account: alice, keyword: 'take') | ||||
|       it 'considers word boundaries when matching' do | ||||
|         KeywordMute.create!(account: alice, keyword: 'bob', whole_word: true) | ||||
| 
 | ||||
|         expect(matcher =~ 'This is a shiitake mushroom').to be_falsy | ||||
|         expect(matcher =~ 'bobcats').to be_falsy | ||||
|       end | ||||
| 
 | ||||
|       it 'matches substrings if whole_word is false' do | ||||
|         KeywordMute.create!(account: alice, keyword: 'take', whole_word: false) | ||||
| 
 | ||||
|         expect(matcher =~ 'This is a shiitake mushroom').to be_truthy | ||||
|       end | ||||
| 
 | ||||
|       it 'matches keywords at the beginning of the text' do | ||||
|  |  | |||
		Loading…
	
		Reference in a new issue