Merge remote-tracking branch 'VyrCossont/search-all-visible-toots'

This reverts commit 93a73ce9db.

app/chewy/statuses_index.rb

@@ -65,7 +65,12 @@ class StatusesIndex < Chewy::Index
   root date_detection: false do
     field :id, type: 'long'
     field :account_id, type: 'long'
+    field :created_at, type: 'date'
     field :visibility, type: 'keyword'
+    field :domain, type: 'keyword', value: ->(status) { status.account.domain or Rails.configuration.x.local_domain }
+    field :lang, type: 'keyword', value: ->(status) { status.language }
+    field :is, type: 'keyword', value: ->(status) { status.searchable_is }
+    field :has, type: 'keyword', value: ->(status) { status.searchable_has }
 
     field :text, type: 'text', value: ->(status) { status.searchable_text } do
       field :stemmed, type: 'text', analyzer: 'content'

app/lib/activitypub/activity/create.rb

@@ -85,7 +85,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
       attach_tags(@status)
     end
 
-    StatusesIndex.import! @status
+    StatusesIndex.import! @status if Chewy.enabled?
 
     resolve_thread(@status)
     fetch_replies(@status)

app/lib/search_query_parser.rb

@@ -9,7 +9,7 @@ class SearchQueryParser < Parslet::Parser
   rule(:prefix)    { (term >> colon).as(:prefix) }
   rule(:shortcode) { (colon >> term >> colon.maybe).as(:shortcode) }
   rule(:phrase)    { (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) }
-  rule(:clause)    { (prefix.maybe >> operator.maybe >> (phrase | term | shortcode)).as(:clause) }
+  rule(:clause)    { (operator.maybe >> prefix.maybe >> (phrase | term | shortcode)).as(:clause) }
   rule(:query)     { (clause >> space.maybe).repeat.as(:query) }
   root(:query)
 end

app/lib/search_query_transformer.rb

@@ -2,7 +2,7 @@
 
 class SearchQueryTransformer < Parslet::Transform
   class Query
-    attr_reader :should_clauses, :must_not_clauses, :must_clauses, :filter_clauses
+    attr_reader :should_clauses, :must_not_clauses, :must_clauses, :filter_clauses, :order_clauses
 
     def initialize(clauses)
       grouped = clauses.chunk(&:operator).to_h
@@ -10,13 +10,20 @@ class SearchQueryTransformer < Parslet::Transform
       @must_not_clauses = grouped.fetch(:must_not, [])
       @must_clauses = grouped.fetch(:must, [])
       @filter_clauses = grouped.fetch(:filter, [])
+      @order_clauses = grouped.fetch(:order, [])
     end
 
-    def apply(search)
+    def apply(search, account)
       should_clauses.each { |clause| search = search.query.should(clause_to_query(clause)) }
       must_clauses.each { |clause| search = search.query.must(clause_to_query(clause)) }
       must_not_clauses.each { |clause| search = search.query.must_not(clause_to_query(clause)) }
-      filter_clauses.each { |clause| search = search.filter(**clause_to_filter(clause)) }
+      filter_clauses.each { |clause| search = search.filter(**clause_to_filter(clause, account)) }
+      if order_clauses.empty?
+        # Default to most recent results first.
+        search = search.order(created_at: :desc)
+      else
+        order_clauses.each { |clause| search = search.order(**clause_to_order(clause)) }
+      end
       search.query.minimum_should_match(1)
     end
 
@@ -28,15 +35,26 @@ class SearchQueryTransformer < Parslet::Transform
         { multi_match: { type: 'most_fields', query: clause.term, fields: ['text', 'text.stemmed'] } }
       when PhraseClause
         { match_phrase: { text: { query: clause.phrase } } }
+      when PrefixClause
+        { term: { clause.filter => clause.term } }
       else
         raise "Unexpected clause type: #{clause}"
       end
     end
 
-    def clause_to_filter(clause)
+    def clause_to_filter(clause, account)
       case clause
       when PrefixClause
-        { term: { clause.filter => clause.term } }
+        { clause.query => { clause.filter => clause.term == :account_id_placeholder ? account.id : clause.term } }
+      else
+        raise "Unexpected clause type: #{clause}"
+      end
+    end
+
+    def clause_to_order(clause)
+      case clause
+      when PrefixClause
+        { clause.term => clause.order }
       else
         raise "Unexpected clause type: #{clause}"
       end
@@ -81,11 +99,35 @@ class SearchQueryTransformer < Parslet::Transform
   end
 
   class PrefixClause
-    attr_reader :filter, :operator, :term
+    attr_reader :filter, :operator, :term, :order, :query
+
+    def initialize(prefix, operator, term)
+      @query = :term
+
+      case operator
+      when '+', nil
+        @operator = :filter
+      when '-'
+        @operator = :must_not
+      else
+        raise "Unknown operator: #{str}"
+      end
 
-    def initialize(prefix, term)
-      @operator = :filter
       case prefix
+      when 'domain', 'is', 'has', 'lang', 'visibility'
+        @filter = prefix.to_s
+        @term = term
+      when 'before', 'after'
+        @query = :range
+        @filter = 'created_at'
+        case prefix
+        when 'before'
+          @term = { lt: term }
+        when 'after'
+          @term = { gt: term }
+        else
+          raise Mastodon::SyntaxError
+        end
       when 'from'
         @filter = :account_id
 
@@ -94,6 +136,25 @@ class SearchQueryTransformer < Parslet::Transform
         account          = Account.find_remote!(username, domain)
 
         @term = account.id
+      when 'scope'
+        raise Mastodon::SyntaxError unless operator.nil?
+        raise Mastodon::SyntaxError unless term == 'classic'
+        @filter = 'searchable_by'
+        @term = :account_id_placeholder
+      when 'sort'
+        raise Mastodon::SyntaxError unless operator.nil?
+
+        @operator = :order
+        @term = :created_at
+
+        case term
+        when 'oldest'
+          @order = :asc
+        when 'newest'
+          @order = :desc
+        else
+          raise Mastodon::SyntaxError
+        end
       else
         raise Mastodon::SyntaxError
       end
@@ -105,7 +166,7 @@ class SearchQueryTransformer < Parslet::Transform
     operator = clause[:operator]&.to_s
 
     if clause[:prefix]
-      PrefixClause.new(prefix, clause[:term].to_s)
+      PrefixClause.new(prefix, operator, clause[:term].to_s)
     elsif clause[:term]
       TermClause.new(prefix, operator, clause[:term].to_s)
     elsif clause[:shortcode]

app/models/status.rb

@@ -189,6 +189,27 @@ class Status < ApplicationRecord
     ].compact.join("\n\n")
   end
 
+  def searchable_is
+    keywords = []
+    keywords << :bot if account.bot?
+    keywords << :local if local?
+    keywords << :local_only if local_only
+    keywords << :reply if reply?
+    keywords << :sensitive if sensitive?
+    keywords
+  end
+
+  def searchable_has
+    keywords = []
+    keywords << :cw if spoiler_text?
+    keywords << :link if FetchLinkCardService.new.link?(self)
+    keywords << :media if media_attachments.present?
+    keywords << :mention if mentions.present?
+    keywords << :poll if preloadable_poll.present?
+    keywords << :tag if tags.present?
+    keywords
+  end
+
   def to_log_human_identifier
     account.acct
   end

app/services/activitypub/process_account_service.rb

@@ -91,7 +91,7 @@ class ActivityPub::ProcessAccountService < BaseService
 
     @account.save_with_optional_media!
 
-    AccountsIndex.import! @account
+    AccountsIndex.import! @account if Chewy.enabled?
   end
 
   def set_immediate_protocol_attributes!

app/services/activitypub/process_status_update_service.rb

@@ -54,7 +54,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
       broadcast_updates!
     end
 
-    StatusesIndex.import! @status
+    StatusesIndex.import! @status if Chewy.enabled?
 
     forward_activity! if significant_changes? && @status_parser.edited_at > last_edit_date
   end

app/services/fetch_link_card_service.rb

@@ -34,6 +34,14 @@ class FetchLinkCardService < BaseService
     nil
   end
 
+  ##
+  # Borrow most of this machinery to detect whether the status has at least one link.
+  def link?(status)
+    @status       = status
+    @original_url = parse_urls
+    !@original_url.nil?
+  end
+
   private
 
   def process_url

app/services/search_service.rb

@@ -42,7 +42,7 @@ class SearchService < BaseService
     when :public_or_unlisted
       statuses_index = statuses_index.filter.or(terms: { visibility: ['public', 'unlisted'] })
     end
-    definition = parsed_query.apply(statuses_index)
+    definition = parsed_query.apply(statuses_index, @account)
 
     if @options[:account_id].present?
       definition = definition.filter(term: { account_id: @options[:account_id] })