diff --git a/.env.production.sample b/.env.production.sample index b604c4b04d..4f21531632 100644 --- a/.env.production.sample +++ b/.env.production.sample @@ -238,7 +238,7 @@ SMTP_FROM_ADDRESS=notifications@example.com # SAML_ATTRIBUTES_STATEMENTS_FULL_NAME="urn:oid:2.16.840.1.113730.3.1.241" # SAML_ATTRIBUTES_STATEMENTS_FIRST_NAME="urn:oid:2.5.4.42" # SAML_ATTRIBUTES_STATEMENTS_LAST_NAME="urn:oid:2.5.4.4" -# SAML_UID_ATTRIBUTE="urn:oid:0.9.2342.19200300.100.1.1" +# SAML_UID_ATTRIBUTE="urn:oid:0.9.2342.19200300.100.1.1" # SAML_ATTRIBUTES_STATEMENTS_VERIFIED= # SAML_ATTRIBUTES_STATEMENTS_VERIFIED_EMAIL= @@ -286,6 +286,12 @@ MAX_REACTIONS=1 # Customize the number of hashtags shown in 'Explore' # MAX_TRENDING_TAGS=10 +# Scope of full-text searches: +# - public: search any status with public visibility +# - public_or_unlisted: search any status with public or unlisted visibility +# - classic: searches only a user's own statuses, favs, bookmarks, and mentions +# SEARCH_SCOPE=public + # Maximum custom emoji file sizes # If undefined or smaller than MAX_EMOJI_SIZE, the value # of MAX_EMOJI_SIZE will be used for MAX_REMOTE_EMOJI_SIZE diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb index 6dd4fb18b0..17df16b612 100644 --- a/app/chewy/statuses_index.rb +++ b/app/chewy/statuses_index.rb @@ -65,6 +65,12 @@ class StatusesIndex < Chewy::Index root date_detection: false do field :id, type: 'long' field :account_id, type: 'long' + field :created_at, type: 'date' + field :visibility, type: 'keyword' + field :domain, type: 'keyword', value: ->(status) { status.account.domain or Rails.configuration.x.local_domain } + field :lang, type: 'keyword', value: ->(status) { status.language } + field :is, type: 'keyword', value: ->(status) { status.searchable_is } + field :has, type: 'keyword', value: ->(status) { status.searchable_has } field :text, type: 'text', value: ->(status) { status.searchable_text } do field :stemmed, type: 'text', analyzer: 'content' diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index faaa968d75..b51e75813c 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -85,6 +85,8 @@ class ActivityPub::Activity::Create < ActivityPub::Activity attach_tags(@status) end + StatusesIndex.import! @status if Chewy.enabled? + resolve_thread(@status) fetch_replies(@status) distribute diff --git a/app/lib/importer/statuses_index_importer.rb b/app/lib/importer/statuses_index_importer.rb index 5b5153d5c8..795bb057c5 100644 --- a/app/lib/importer/statuses_index_importer.rb +++ b/app/lib/importer/statuses_index_importer.rb @@ -25,7 +25,9 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter # on the results of the filter, so this filtering happens here instead bulk.map! do |entry| new_entry = begin - if entry[:index] && entry.dig(:index, :data, 'searchable_by').blank? + if entry[:index] && + entry.dig(:index, :data, 'searchable_by').blank? && + Rails.configuration.x.search_scope == :classic { delete: entry[:index].except(:data) } else entry @@ -58,13 +60,21 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter end def scopes - [ + classic_scopes = [ local_statuses_scope, local_mentions_scope, local_favourites_scope, local_votes_scope, local_bookmarks_scope, ] + case Rails.configuration.x.search_scope + when :public + classic_scopes + [public_scope] + when :public_or_unlisted + classic_scopes + [public_or_unlisted_scope] + else + classic_scopes + end end def local_mentions_scope @@ -86,4 +96,12 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter def local_statuses_scope Status.local.select('"statuses"."id", COALESCE("statuses"."reblog_of_id", "statuses"."id") AS status_id') end + + def public_scope + Status.with_public_visibility.select('"statuses"."id", "statuses"."id" AS status_id') + end + + def public_or_unlisted_scope + Status.with_public_or_unlisted_visibility.select('"statuses"."id", "statuses"."id" AS status_id') + end end diff --git a/app/lib/search_query_parser.rb b/app/lib/search_query_parser.rb index 15956d4cfd..5d6ffbf29d 100644 --- a/app/lib/search_query_parser.rb +++ b/app/lib/search_query_parser.rb @@ -9,7 +9,7 @@ class SearchQueryParser < Parslet::Parser rule(:prefix) { (term >> colon).as(:prefix) } rule(:shortcode) { (colon >> term >> colon.maybe).as(:shortcode) } rule(:phrase) { (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) } - rule(:clause) { (prefix.maybe >> operator.maybe >> (phrase | term | shortcode)).as(:clause) } + rule(:clause) { (operator.maybe >> prefix.maybe >> (phrase | term | shortcode)).as(:clause) } rule(:query) { (clause >> space.maybe).repeat.as(:query) } root(:query) end diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb index aef05e9d9d..a6c0c2f2d4 100644 --- a/app/lib/search_query_transformer.rb +++ b/app/lib/search_query_transformer.rb @@ -2,7 +2,7 @@ class SearchQueryTransformer < Parslet::Transform class Query - attr_reader :should_clauses, :must_not_clauses, :must_clauses, :filter_clauses + attr_reader :should_clauses, :must_not_clauses, :must_clauses, :filter_clauses, :order_clauses def initialize(clauses) grouped = clauses.chunk(&:operator).to_h @@ -10,13 +10,20 @@ class SearchQueryTransformer < Parslet::Transform @must_not_clauses = grouped.fetch(:must_not, []) @must_clauses = grouped.fetch(:must, []) @filter_clauses = grouped.fetch(:filter, []) + @order_clauses = grouped.fetch(:order, []) end - def apply(search) + def apply(search, account) should_clauses.each { |clause| search = search.query.should(clause_to_query(clause)) } must_clauses.each { |clause| search = search.query.must(clause_to_query(clause)) } must_not_clauses.each { |clause| search = search.query.must_not(clause_to_query(clause)) } - filter_clauses.each { |clause| search = search.filter(**clause_to_filter(clause)) } + filter_clauses.each { |clause| search = search.filter(**clause_to_filter(clause, account)) } + if order_clauses.empty? + # Default to most recent results first. + search = search.order(created_at: :desc) + else + order_clauses.each { |clause| search = search.order(**clause_to_order(clause)) } + end search.query.minimum_should_match(1) end @@ -28,15 +35,26 @@ class SearchQueryTransformer < Parslet::Transform { multi_match: { type: 'most_fields', query: clause.term, fields: ['text', 'text.stemmed'] } } when PhraseClause { match_phrase: { text: { query: clause.phrase } } } + when PrefixClause + { term: { clause.filter => clause.term } } else raise "Unexpected clause type: #{clause}" end end - def clause_to_filter(clause) + def clause_to_filter(clause, account) case clause when PrefixClause - { term: { clause.filter => clause.term } } + { clause.query => { clause.filter => clause.term == :account_id_placeholder ? account.id : clause.term } } + else + raise "Unexpected clause type: #{clause}" + end + end + + def clause_to_order(clause) + case clause + when PrefixClause + { clause.term => clause.order } else raise "Unexpected clause type: #{clause}" end @@ -81,11 +99,35 @@ class SearchQueryTransformer < Parslet::Transform end class PrefixClause - attr_reader :filter, :operator, :term + attr_reader :filter, :operator, :term, :order, :query + + def initialize(prefix, operator, term) + @query = :term + + case operator + when '+', nil + @operator = :filter + when '-' + @operator = :must_not + else + raise "Unknown operator: #{str}" + end - def initialize(prefix, term) - @operator = :filter case prefix + when 'domain', 'is', 'has', 'lang', 'visibility' + @filter = prefix.to_s + @term = term + when 'before', 'after' + @query = :range + @filter = 'created_at' + case prefix + when 'before' + @term = { lt: term } + when 'after' + @term = { gt: term } + else + raise Mastodon::SyntaxError + end when 'from' @filter = :account_id @@ -94,6 +136,25 @@ class SearchQueryTransformer < Parslet::Transform account = Account.find_remote!(username, domain) @term = account.id + when 'scope' + raise Mastodon::SyntaxError unless operator.nil? + raise Mastodon::SyntaxError unless term == 'classic' + @filter = 'searchable_by' + @term = :account_id_placeholder + when 'sort' + raise Mastodon::SyntaxError unless operator.nil? + + @operator = :order + @term = :created_at + + case term + when 'oldest' + @order = :asc + when 'newest' + @order = :desc + else + raise Mastodon::SyntaxError + end else raise Mastodon::SyntaxError end @@ -105,7 +166,7 @@ class SearchQueryTransformer < Parslet::Transform operator = clause[:operator]&.to_s if clause[:prefix] - PrefixClause.new(prefix, clause[:term].to_s) + PrefixClause.new(prefix, operator, clause[:term].to_s) elsif clause[:term] TermClause.new(prefix, operator, clause[:term].to_s) elsif clause[:shortcode] diff --git a/app/models/status.rb b/app/models/status.rb index 8f212b9b79..1ae0969724 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -104,6 +104,7 @@ class Status < ApplicationRecord scope :without_replies, -> { where('statuses.reply = FALSE OR statuses.in_reply_to_account_id = statuses.account_id') } scope :without_reblogs, -> { where('statuses.reblog_of_id IS NULL') } scope :with_public_visibility, -> { where(visibility: :public) } + scope :with_public_or_unlisted_visibility, -> { where(visibility: [:public, :unlisted]) } scope :tagged_with, ->(tag_ids) { joins(:statuses_tags).where(statuses_tags: { tag_id: tag_ids }) } scope :excluding_silenced_accounts, -> { left_outer_joins(:account).where(accounts: { silenced_at: nil }) } scope :including_silenced_accounts, -> { left_outer_joins(:account).where.not(accounts: { silenced_at: nil }) } @@ -188,6 +189,27 @@ class Status < ApplicationRecord ].compact.join("\n\n") end + def searchable_is + keywords = [] + keywords << :bot if account.bot? + keywords << :local if local? + keywords << :local_only if local_only + keywords << :reply if reply? + keywords << :sensitive if sensitive? + keywords + end + + def searchable_has + keywords = [] + keywords << :cw if spoiler_text? + keywords << :link if FetchLinkCardService.new.link?(self) + keywords << :media if media_attachments.present? + keywords << :mention if mentions.present? + keywords << :poll if preloadable_poll.present? + keywords << :tag if tags.present? + keywords + end + def to_log_human_identifier account.acct end diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 2da9096c73..44167f97a3 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -90,6 +90,8 @@ class ActivityPub::ProcessAccountService < BaseService set_fetchable_attributes! unless @options[:only_key] || @account.suspended? @account.save_with_optional_media! + + AccountsIndex.import! @account if Chewy.enabled? end def set_immediate_protocol_attributes! diff --git a/app/services/activitypub/process_status_update_service.rb b/app/services/activitypub/process_status_update_service.rb index 11b38ab92b..d6f1f3e5d2 100644 --- a/app/services/activitypub/process_status_update_service.rb +++ b/app/services/activitypub/process_status_update_service.rb @@ -54,6 +54,8 @@ class ActivityPub::ProcessStatusUpdateService < BaseService broadcast_updates! end + StatusesIndex.import! @status if Chewy.enabled? + forward_activity! if significant_changes? && @status_parser.edited_at > last_edit_date end diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index e5c0448925..3853339b64 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -34,6 +34,14 @@ class FetchLinkCardService < BaseService nil end + ## + # Borrow most of this machinery to detect whether the status has at least one link. + def link?(status) + @status = status + @original_url = parse_urls + !@original_url.nil? + end + private def process_url diff --git a/app/services/search_service.rb b/app/services/search_service.rb index 1a76cbb388..673cf34b82 100644 --- a/app/services/search_service.rb +++ b/app/services/search_service.rb @@ -35,7 +35,14 @@ class SearchService < BaseService end def perform_statuses_search! - definition = parsed_query.apply(StatusesIndex.filter(term: { searchable_by: @account.id })) + statuses_index = StatusesIndex.filter(term: { searchable_by: @account.id }) + case Rails.configuration.x.search_scope + when :public + statuses_index = statuses_index.filter.or(term: { visibility: 'public' }) + when :public_or_unlisted + statuses_index = statuses_index.filter.or(terms: { visibility: ['public', 'unlisted'] }) + end + definition = parsed_query.apply(statuses_index, @account) if @options[:account_id].present? definition = definition.filter(term: { account_id: @options[:account_id] }) diff --git a/config/initializers/search_scope.rb b/config/initializers/search_scope.rb new file mode 100644 index 0000000000..3024405709 --- /dev/null +++ b/config/initializers/search_scope.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +Rails.application.configure do + config.x.search_scope = case + when ENV['SEARCH_SCOPE'] == 'public' + :public + when ENV['SEARCH_SCOPE'] == 'public_or_unlisted' + :public_or_unlisted + else + :classic + end +end