Compare commits

...

22 commits

Author SHA1 Message Date
388300482c
Merge remote-tracking branch 'VyrCossont/search-all-visible-toots' 2023-02-04 19:57:12 +09:00
919292c7c3
Revert "Extended Search" to add in properly
This reverts commit 93a73ce9db.
2023-02-04 19:56:07 +09:00
Vyr Cossont
b342b12dbd Don't import if ES is not enabled 2023-01-30 21:18:15 -08:00
Vyr Cossont
d1797b437f Add scope:classic filter 2023-01-01 16:31:40 -08:00
Vyr Cossont
30faf1f6da Add before: and after: filters 2023-01-01 15:30:13 -08:00
Vyr Cossont
a6ee508eb7 Add visibility: and is:sensitive filters 2022-12-30 15:58:34 -08:00
Vyr Cossont
bf34294fcb Add is:, has:, domain:, lang: search filters 2022-12-30 15:11:27 -08:00
Vyr Cossont
26d465175a Add sort operator, default to newest first 2022-12-30 12:48:23 -08:00
Vyr Cossont
40519dac1a Remove index fields that are not yet used 2022-12-19 13:24:43 -08:00
Vyr Cossont
3ea4a5ed29 Fix rubocop complaint 2022-12-19 13:17:29 -08:00
Vyr Cossont
844a1b8476 parallel gem no longer required 2022-12-19 13:17:12 -08:00
Vyr Cossont
9474bb27fc The id column is actually used: something in the bulk import requires that the primary key column be in the custom select 2022-12-18 22:39:08 -08:00
Vyr Cossont
ac891d9dce Don't update status inside a lock, it's just queueing anyway 2022-12-18 16:34:07 -08:00
Vyr Cossont
87b5f7f485 Explicit ES import when AP status or account is created or updated 2022-12-18 16:21:32 -08:00
Vyr Cossont
98c9bc52c0 Update indexer, add scope switch 2022-12-11 14:24:02 -08:00
Vyr Cossont
494d98c3a8 Fix incomplete name change 2022-11-28 23:26:59 -08:00
Vyr Cossont
3fa72fd56c Don't search unlisted statuses 2022-11-28 22:53:51 -08:00
Vyr Cossont
a3faf83586 Fix rubocop lint 2022-11-28 20:15:07 -08:00
Vyr Cossont
b831c3897d Fix transposition of search DSL calls 2022-11-28 20:06:19 -08:00
Vyr Cossont
d3c1eaba21 Change TOOTS to STATUSES 2022-11-28 18:45:20 -08:00
Vyr Cossont
f6f13f54cf Add dependency for Chewy parallel reindexing 2022-11-28 11:05:17 -08:00
Vyr Cossont
60654e8111 Add SEARCH_ALL_VISIBLE_TOOTS env flag
Context: https://docs.joinmastodon.org/user/network/#search

Vanilla Mastodon intentionally refuses to search outside a user's
own toots, favs, bookmarks, and mentions. This flag makes that
restricted search behavior a per-instance choice, defaulting
to the same behavior as vanilla Mastodon if the flag is absent.
2022-11-28 10:47:58 -08:00
9 changed files with 109 additions and 14 deletions

View file

@ -65,7 +65,12 @@ class StatusesIndex < Chewy::Index
root date_detection: false do
field :id, type: 'long'
field :account_id, type: 'long'
field :created_at, type: 'date'
field :visibility, type: 'keyword'
field :domain, type: 'keyword', value: ->(status) { status.account.domain or Rails.configuration.x.local_domain }
field :lang, type: 'keyword', value: ->(status) { status.language }
field :is, type: 'keyword', value: ->(status) { status.searchable_is }
field :has, type: 'keyword', value: ->(status) { status.searchable_has }
field :text, type: 'text', value: ->(status) { status.searchable_text } do
field :stemmed, type: 'text', analyzer: 'content'

View file

@ -85,7 +85,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
attach_tags(@status)
end
StatusesIndex.import! @status
StatusesIndex.import! @status if Chewy.enabled?
resolve_thread(@status)
fetch_replies(@status)

View file

@ -9,7 +9,7 @@ class SearchQueryParser < Parslet::Parser
rule(:prefix) { (term >> colon).as(:prefix) }
rule(:shortcode) { (colon >> term >> colon.maybe).as(:shortcode) }
rule(:phrase) { (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) }
rule(:clause) { (prefix.maybe >> operator.maybe >> (phrase | term | shortcode)).as(:clause) }
rule(:clause) { (operator.maybe >> prefix.maybe >> (phrase | term | shortcode)).as(:clause) }
rule(:query) { (clause >> space.maybe).repeat.as(:query) }
root(:query)
end

View file

@ -2,7 +2,7 @@
class SearchQueryTransformer < Parslet::Transform
class Query
attr_reader :should_clauses, :must_not_clauses, :must_clauses, :filter_clauses
attr_reader :should_clauses, :must_not_clauses, :must_clauses, :filter_clauses, :order_clauses
def initialize(clauses)
grouped = clauses.chunk(&:operator).to_h
@ -10,13 +10,20 @@ class SearchQueryTransformer < Parslet::Transform
@must_not_clauses = grouped.fetch(:must_not, [])
@must_clauses = grouped.fetch(:must, [])
@filter_clauses = grouped.fetch(:filter, [])
@order_clauses = grouped.fetch(:order, [])
end
def apply(search)
def apply(search, account)
should_clauses.each { |clause| search = search.query.should(clause_to_query(clause)) }
must_clauses.each { |clause| search = search.query.must(clause_to_query(clause)) }
must_not_clauses.each { |clause| search = search.query.must_not(clause_to_query(clause)) }
filter_clauses.each { |clause| search = search.filter(**clause_to_filter(clause)) }
filter_clauses.each { |clause| search = search.filter(**clause_to_filter(clause, account)) }
if order_clauses.empty?
# Default to most recent results first.
search = search.order(created_at: :desc)
else
order_clauses.each { |clause| search = search.order(**clause_to_order(clause)) }
end
search.query.minimum_should_match(1)
end
@ -28,15 +35,26 @@ class SearchQueryTransformer < Parslet::Transform
{ multi_match: { type: 'most_fields', query: clause.term, fields: ['text', 'text.stemmed'] } }
when PhraseClause
{ match_phrase: { text: { query: clause.phrase } } }
when PrefixClause
{ term: { clause.filter => clause.term } }
else
raise "Unexpected clause type: #{clause}"
end
end
def clause_to_filter(clause)
def clause_to_filter(clause, account)
case clause
when PrefixClause
{ term: { clause.filter => clause.term } }
{ clause.query => { clause.filter => clause.term == :account_id_placeholder ? account.id : clause.term } }
else
raise "Unexpected clause type: #{clause}"
end
end
def clause_to_order(clause)
case clause
when PrefixClause
{ clause.term => clause.order }
else
raise "Unexpected clause type: #{clause}"
end
@ -81,11 +99,35 @@ class SearchQueryTransformer < Parslet::Transform
end
class PrefixClause
attr_reader :filter, :operator, :term
attr_reader :filter, :operator, :term, :order, :query
def initialize(prefix, operator, term)
@query = :term
case operator
when '+', nil
@operator = :filter
when '-'
@operator = :must_not
else
raise "Unknown operator: #{str}"
end
def initialize(prefix, term)
@operator = :filter
case prefix
when 'domain', 'is', 'has', 'lang', 'visibility'
@filter = prefix.to_s
@term = term
when 'before', 'after'
@query = :range
@filter = 'created_at'
case prefix
when 'before'
@term = { lt: term }
when 'after'
@term = { gt: term }
else
raise Mastodon::SyntaxError
end
when 'from'
@filter = :account_id
@ -94,6 +136,25 @@ class SearchQueryTransformer < Parslet::Transform
account = Account.find_remote!(username, domain)
@term = account.id
when 'scope'
raise Mastodon::SyntaxError unless operator.nil?
raise Mastodon::SyntaxError unless term == 'classic'
@filter = 'searchable_by'
@term = :account_id_placeholder
when 'sort'
raise Mastodon::SyntaxError unless operator.nil?
@operator = :order
@term = :created_at
case term
when 'oldest'
@order = :asc
when 'newest'
@order = :desc
else
raise Mastodon::SyntaxError
end
else
raise Mastodon::SyntaxError
end
@ -105,7 +166,7 @@ class SearchQueryTransformer < Parslet::Transform
operator = clause[:operator]&.to_s
if clause[:prefix]
PrefixClause.new(prefix, clause[:term].to_s)
PrefixClause.new(prefix, operator, clause[:term].to_s)
elsif clause[:term]
TermClause.new(prefix, operator, clause[:term].to_s)
elsif clause[:shortcode]

View file

@ -189,6 +189,27 @@ class Status < ApplicationRecord
].compact.join("\n\n")
end
def searchable_is
keywords = []
keywords << :bot if account.bot?
keywords << :local if local?
keywords << :local_only if local_only
keywords << :reply if reply?
keywords << :sensitive if sensitive?
keywords
end
def searchable_has
keywords = []
keywords << :cw if spoiler_text?
keywords << :link if FetchLinkCardService.new.link?(self)
keywords << :media if media_attachments.present?
keywords << :mention if mentions.present?
keywords << :poll if preloadable_poll.present?
keywords << :tag if tags.present?
keywords
end
def to_log_human_identifier
account.acct
end

View file

@ -91,7 +91,7 @@ class ActivityPub::ProcessAccountService < BaseService
@account.save_with_optional_media!
AccountsIndex.import! @account
AccountsIndex.import! @account if Chewy.enabled?
end
def set_immediate_protocol_attributes!

View file

@ -54,7 +54,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
broadcast_updates!
end
StatusesIndex.import! @status
StatusesIndex.import! @status if Chewy.enabled?
forward_activity! if significant_changes? && @status_parser.edited_at > last_edit_date
end

View file

@ -34,6 +34,14 @@ class FetchLinkCardService < BaseService
nil
end
##
# Borrow most of this machinery to detect whether the status has at least one link.
def link?(status)
@status = status
@original_url = parse_urls
!@original_url.nil?
end
private
def process_url

View file

@ -42,7 +42,7 @@ class SearchService < BaseService
when :public_or_unlisted
statuses_index = statuses_index.filter.or(terms: { visibility: ['public', 'unlisted'] })
end
definition = parsed_query.apply(statuses_index)
definition = parsed_query.apply(statuses_index, @account)
if @options[:account_id].present?
definition = definition.filter(term: { account_id: @options[:account_id] })