Merge branch 'master' into glitch-soc/merge-upstream

.env.production.sample

@@ -88,6 +88,10 @@ SMTP_FROM_ADDRESS=notifications@example.com
 # CDN_HOST=https://assets.example.com
 
 # S3 (optional)
+# The attachment host must allow cross origin request from WEB_DOMAIN or
+# LOCAL_DOMAIN if WEB_DOMAIN is not set. For example, the server may have the
+# following header field:
+# Access-Control-Allow-Origin: https://192.168.1.123:9000/
 # S3_ENABLED=true
 # S3_BUCKET=
 # AWS_ACCESS_KEY_ID=
@@ -97,6 +101,8 @@ SMTP_FROM_ADDRESS=notifications@example.com
 # S3_HOSTNAME=192.168.1.123:9000
 
 # S3 (Minio Config (optional) Please check Minio instance for details)
+# The attachment host must allow cross origin request - see the description
+# above.
 # S3_ENABLED=true
 # S3_BUCKET=
 # AWS_ACCESS_KEY_ID=
@@ -108,6 +114,8 @@ SMTP_FROM_ADDRESS=notifications@example.com
 # S3_SIGNATURE_VERSION=
 
 # Swift (optional)
+# The attachment host must allow cross origin request - see the description
+# above.
 # SWIFT_ENABLED=true
 # SWIFT_USERNAME=
 # For Keystone V3, the value for SWIFT_TENANT should be the project name

.eslintrc.yml

@@ -7,6 +7,9 @@ env:
   es6: true
   jest: true
 
+globals:
+  ATTACHMENT_HOST: false
+
 parser: babel-eslint
 
 plugins:

app/controllers/api/v1/timelines/direct_controller.rb

@@ -23,15 +23,18 @@ class Api::V1::Timelines::DirectController < Api::BaseController
   end
 
   def direct_statuses
-    direct_timeline_statuses.paginate_by_max_id(
-      limit_param(DEFAULT_STATUSES_LIMIT),
-      params[:max_id],
-      params[:since_id]
-    )
+    direct_timeline_statuses
   end
 
   def direct_timeline_statuses
-    Status.as_direct_timeline(current_account)
+    # this query requires built in pagination.
+    Status.as_direct_timeline(
+      current_account,
+      limit_param(DEFAULT_STATUSES_LIMIT),
+      params[:max_id],
+      params[:since_id],
+      true # returns array of cache_ids object
+    )
   end
 
   def insert_pagination_headers

app/controllers/api/v1/trends_controller.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class Api::V1::TrendsController < Api::BaseController
+  before_action :set_tags
+
+  respond_to :json
+
+  def index
+    render json: @tags, each_serializer: REST::TagSerializer
+  end
+
+  private
+
+  def set_tags
+    @tags = TrendingTags.get(limit_param(10))
+  end
+end

app/controllers/api/v2/search_controller.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class Api::V2::SearchController < Api::V1::SearchController
+  def index
+    @search = Search.new(search)
+    render json: @search, serializer: REST::V2::SearchSerializer
+  end
+end

app/javascript/mastodon/actions/search.js

@@ -33,7 +33,7 @@ export function submitSearch() {
 
     dispatch(fetchSearchRequest());
 
-    api(getState).get('/api/v1/search', {
+    api(getState).get('/api/v2/search', {
       params: {
         q: value,
         resolve: true,

app/javascript/mastodon/actions/trends.js

@@ -0,0 +1,32 @@
+import api from '../api';
+
+export const TRENDS_FETCH_REQUEST = 'TRENDS_FETCH_REQUEST';
+export const TRENDS_FETCH_SUCCESS = 'TRENDS_FETCH_SUCCESS';
+export const TRENDS_FETCH_FAIL    = 'TRENDS_FETCH_FAIL';
+
+export const fetchTrends = () => (dispatch, getState) => {
+  dispatch(fetchTrendsRequest());
+
+  api(getState)
+    .get('/api/v1/trends')
+    .then(({ data }) => dispatch(fetchTrendsSuccess(data)))
+    .catch(err => dispatch(fetchTrendsFail(err)));
+};
+
+export const fetchTrendsRequest = () => ({
+  type: TRENDS_FETCH_REQUEST,
+  skipLoading: true,
+});
+
+export const fetchTrendsSuccess = trends => ({
+  type: TRENDS_FETCH_SUCCESS,
+  trends,
+  skipLoading: true,
+});
+
+export const fetchTrendsFail = error => ({
+  type: TRENDS_FETCH_FAIL,
+  error,
+  skipLoading: true,
+  skipAlert: true,
+});

app/javascript/mastodon/components/scrollable_list.js

@@ -25,6 +25,7 @@ export default class ScrollableList extends PureComponent {
     isLoading: PropTypes.bool,
     hasMore: PropTypes.bool,
     prepend: PropTypes.node,
+    alwaysPrepend: PropTypes.bool,
     emptyMessage: PropTypes.node,
     children: PropTypes.node,
   };
@@ -140,7 +141,7 @@ export default class ScrollableList extends PureComponent {
   }
 
   render () {
-    const { children, scrollKey, trackScroll, shouldUpdateScroll, isLoading, hasMore, prepend, emptyMessage, onLoadMore } = this.props;
+    const { children, scrollKey, trackScroll, shouldUpdateScroll, isLoading, hasMore, prepend, alwaysPrepend, emptyMessage, onLoadMore } = this.props;
     const { fullscreen } = this.state;
     const childrenCount = React.Children.count(children);
 
@@ -172,9 +173,13 @@ export default class ScrollableList extends PureComponent {
       );
     } else {
       scrollableArea = (
+        <div style={{ flex: '1 1 auto', display: 'flex', flexDirection: 'column' }}>
+          {alwaysPrepend && prepend}
+
           <div className='empty-column-indicator' ref={this.setRef}>
             {emptyMessage}
           </div>
+        </div>
       );
     }
 

app/javascript/mastodon/components/status_list.js

@@ -24,6 +24,7 @@ export default class StatusList extends ImmutablePureComponent {
     hasMore: PropTypes.bool,
     prepend: PropTypes.node,
     emptyMessage: PropTypes.node,
+    alwaysPrepend: PropTypes.bool,
   };
 
   static defaultProps = {

app/javascript/mastodon/features/community_timeline/index.js

@@ -127,6 +127,7 @@ export default class CommunityTimeline extends React.PureComponent {
 
         <StatusListContainer
           prepend={headline}
+          alwaysPrepend
           trackScroll={!pinned}
           scrollKey={`community_timeline-${columnId}`}
           timelineId={`community${onlyMedia ? ':media' : ''}`}

app/javascript/mastodon/features/compose/components/search_results.js

@@ -1,28 +1,82 @@
 import React from 'react';
+import PropTypes from 'prop-types';
 import ImmutablePropTypes from 'react-immutable-proptypes';
-import { FormattedMessage } from 'react-intl';
+import { FormattedMessage, FormattedNumber } from 'react-intl';
 import AccountContainer from '../../../containers/account_container';
 import StatusContainer from '../../../containers/status_container';
 import { Link } from 'react-router-dom';
 import ImmutablePureComponent from 'react-immutable-pure-component';
+import { Sparklines, SparklinesCurve } from 'react-sparklines';
+
+const shortNumberFormat = number => {
+  if (number < 1000) {
+    return <FormattedNumber value={number} />;
+  } else {
+    return <React.Fragment><FormattedNumber value={number / 1000} maximumFractionDigits={1} />K</React.Fragment>;
+  }
+};
+
+const renderHashtag = hashtag => (
+  <div className='trends__item' key={hashtag.get('name')}>
+    <div className='trends__item__name'>
+      <Link to={`/timelines/tag/${hashtag.get('name')}`}>
+        #<span>{hashtag.get('name')}</span>
+      </Link>
+
+      <FormattedMessage id='trends.count_by_accounts' defaultMessage='{count} {rawCount, plural, one {person} other {people}} talking' values={{ rawCount: hashtag.getIn(['history', 0, 'accounts']), count: <strong>{shortNumberFormat(hashtag.getIn(['history', 0, 'accounts']))}</strong> }} />
+    </div>
+
+    <div className='trends__item__current'>
+      {shortNumberFormat(hashtag.getIn(['history', 0, 'uses']))}
+    </div>
+
+    <div className='trends__item__sparkline'>
+      <Sparklines width={50} height={28} data={hashtag.get('history').reverse().map(day => day.get('uses')).toArray()}>
+        <SparklinesCurve style={{ fill: 'none' }} />
+      </Sparklines>
+    </div>
+  </div>
+);
 
 export default class SearchResults extends ImmutablePureComponent {
 
   static propTypes = {
     results: ImmutablePropTypes.map.isRequired,
+    trends: ImmutablePropTypes.list,
+    fetchTrends: PropTypes.func.isRequired,
   };
 
+  componentDidMount () {
+    const { fetchTrends } = this.props;
+    fetchTrends();
+  }
+
   render () {
-    const { results } = this.props;
+    const { results, trends } = this.props;
 
     let accounts, statuses, hashtags;
     let count = 0;
 
+    if (results.isEmpty()) {
+      return (
+        <div className='search-results'>
+          <div className='trends'>
+            <div className='trends__header'>
+              <i className='fa fa-fire fa-fw' />
+              <FormattedMessage id='trends.header' defaultMessage='Trending now' />
+            </div>
+
+            {trends && trends.map(hashtag => renderHashtag(hashtag))}
+          </div>
+        </div>
+      );
+    }
+
     if (results.get('accounts') && results.get('accounts').size > 0) {
       count   += results.get('accounts').size;
       accounts = (
         <div className='search-results__section'>
-          <h5><FormattedMessage id='search_results.accounts' defaultMessage='People' /></h5>
+          <h5><i className='fa fa-fw fa-users' /><FormattedMessage id='search_results.accounts' defaultMessage='People' /></h5>
 
           {results.get('accounts').map(accountId => <AccountContainer key={accountId} id={accountId} />)}
         </div>
@@ -33,7 +87,7 @@ export default class SearchResults extends ImmutablePureComponent {
       count   += results.get('statuses').size;
       statuses = (
         <div className='search-results__section'>
-          <h5><FormattedMessage id='search_results.statuses' defaultMessage='Toots' /></h5>
+          <h5><i className='fa fa-fw fa-quote-right' /><FormattedMessage id='search_results.statuses' defaultMessage='Toots' /></h5>
 
           {results.get('statuses').map(statusId => <StatusContainer key={statusId} id={statusId} />)}
         </div>
@@ -44,13 +98,9 @@ export default class SearchResults extends ImmutablePureComponent {
       count += results.get('hashtags').size;
       hashtags = (
         <div className='search-results__section'>
-          <h5><FormattedMessage id='search_results.hashtags' defaultMessage='Hashtags' /></h5>
+          <h5><i className='fa fa-fw fa-hashtag' /><FormattedMessage id='search_results.hashtags' defaultMessage='Hashtags' /></h5>
 
-          {results.get('hashtags').map(hashtag => (
-            <Link key={hashtag} className='search-results__hashtag' to={`/timelines/tag/${hashtag}`}>
-              #{hashtag}
-            </Link>
-          ))}
+          {results.get('hashtags').map(hashtag => renderHashtag(hashtag))}
         </div>
       );
     }
@@ -58,6 +108,7 @@ export default class SearchResults extends ImmutablePureComponent {
     return (
       <div className='search-results'>
         <div className='search-results__header'>
+          <i className='fa fa-search fa-fw' />
           <FormattedMessage id='search_results.total' defaultMessage='{count, number} {count, plural, one {result} other {results}}' values={{ count }} />
         </div>
 

app/javascript/mastodon/features/compose/containers/search_results_container.js

@@ -1,8 +1,14 @@
 import { connect } from 'react-redux';
 import SearchResults from '../components/search_results';
+import { fetchTrends } from '../../../actions/trends';
 
 const mapStateToProps = state => ({
   results: state.getIn(['search', 'results']),
+  trends: state.get('trends'),
 });
 
-export default connect(mapStateToProps)(SearchResults);
+const mapDispatchToProps = dispatch => ({
+  fetchTrends: () => dispatch(fetchTrends()),
+});
+
+export default connect(mapStateToProps, mapDispatchToProps)(SearchResults);

app/javascript/mastodon/features/compose/index.js

@@ -101,7 +101,7 @@ export default class Compose extends React.PureComponent {
         {(multiColumn || isSearchPage) && <SearchContainer /> }
 
         <div className='drawer__pager'>
-          <div className='drawer__inner' onFocus={this.onFocus}>
+          {!isSearchPage && <div className='drawer__inner' onFocus={this.onFocus}>
             <NavigationContainer onClose={this.onBlur} />
             <ComposeFormContainer />
             {multiColumn && (
@@ -109,7 +109,7 @@ export default class Compose extends React.PureComponent {
                 <img alt='' draggable='false' src={elephantUIPlane} />
               </div>
             )}
-          </div>
+          </div>}
 
           <Motion defaultStyle={{ x: isSearchPage ? 0 : -100 }} style={{ x: spring(showSearch || isSearchPage ? 0 : -100, { stiffness: 210, damping: 20 }) }}>
             {({ x }) => (

app/javascript/mastodon/features/public_timeline/index.js

@@ -127,6 +127,7 @@ export default class PublicTimeline extends React.PureComponent {
 
         <StatusListContainer
           prepend={headline}
+          alwaysPrepend
           timelineId={`public${onlyMedia ? ':media' : ''}`}
           onLoadMore={this.handleLoadMore}
           trackScroll={!pinned}

app/javascript/mastodon/features/status/index.js

@@ -62,31 +62,28 @@ const makeMapStateToProps = () => {
 
     if (status) {
       ancestorsIds = ancestorsIds.withMutations(mutable => {
-        function addAncestor(id) {
-          if (id) {
-            const inReplyTo = state.getIn(['contexts', 'inReplyTos', id]);
+        let id = status.get('in_reply_to_id');
 
+        while (id) {
           mutable.unshift(id);
-            addAncestor(inReplyTo);
+          id = state.getIn(['contexts', 'inReplyTos', id]);
         }
-        }
-
-        addAncestor(status.get('in_reply_to_id'));
       });
 
       descendantsIds = descendantsIds.withMutations(mutable => {
-        function addDescendantOf(id) {
+        const ids = [status.get('id')];
+
+        while (ids.length > 0) {
+          let id        = ids.shift();
           const replies = state.getIn(['contexts', 'replies', id]);
 
           if (replies) {
             replies.forEach(reply => {
               mutable.push(reply);
-              addDescendantOf(reply);
+              ids.unshift(reply);
             });
           }
         }
-
-        addDescendantOf(status.get('id'));
       });
     }
 

app/javascript/mastodon/reducers/index.js

@@ -26,6 +26,7 @@ import height_cache from './height_cache';
 import custom_emojis from './custom_emojis';
 import lists from './lists';
 import listEditor from './list_editor';
+import trends from './trends';
 
 const reducers = {
   dropdown_menu,
@@ -55,6 +56,7 @@ const reducers = {
   custom_emojis,
   lists,
   listEditor,
+  trends,
 };
 
 export default combineReducers(reducers);

app/javascript/mastodon/reducers/search.js

@@ -9,7 +9,7 @@ import {
   COMPOSE_REPLY,
   COMPOSE_DIRECT,
 } from '../actions/compose';
-import { Map as ImmutableMap, List as ImmutableList } from 'immutable';
+import { Map as ImmutableMap, List as ImmutableList, fromJS } from 'immutable';
 
 const initialState = ImmutableMap({
   value: '',
@@ -39,7 +39,7 @@ export default function search(state = initialState, action) {
     return state.set('results', ImmutableMap({
       accounts: ImmutableList(action.results.accounts.map(item => item.id)),
       statuses: ImmutableList(action.results.statuses.map(item => item.id)),
-      hashtags: ImmutableList(action.results.hashtags),
+      hashtags: fromJS(action.results.hashtags),
     })).set('submitted', true);
   default:
     return state;

app/javascript/mastodon/reducers/trends.js

@@ -0,0 +1,13 @@
+import { TRENDS_FETCH_SUCCESS } from '../actions/trends';
+import { fromJS } from 'immutable';
+
+const initialState = null;
+
+export default function trendsReducer(state = initialState, action) {
+  switch(action.type) {
+  case TRENDS_FETCH_SUCCESS:
+    return fromJS(action.trends);
+  default:
+    return state;
+  }
+};

app/javascript/mastodon/service_worker/entry.js

@@ -49,7 +49,7 @@ self.addEventListener('fetch', function(event) {
 
       return response;
     }));
-  } else if (storageFreeable && process.env.CDN_HOST ? url.host === process.env.CDN_HOST : url.pathname.startsWith('/system/')) {
+  } else if (storageFreeable && (ATTACHMENT_HOST ? url.host === ATTACHMENT_HOST : url.pathname.startsWith('/system/'))) {
     event.respondWith(openSystemCache().then(cache => {
       return cache.match(event.request.url).then(cached => {
         if (cached === undefined) {

app/javascript/styles/mastodon-light/diff.scss

@@ -132,20 +132,54 @@
 .boost-modal,
 .confirmation-modal,
 .mute-modal,
-.report-modal {
+.report-modal,
+.embed-modal,
+.error-modal,
+.onboarding-modal {
   background: $ui-base-color;
 }
 
 .boost-modal__action-bar,
 .confirmation-modal__action-bar,
-.mute-modal__action-bar {
+.mute-modal__action-bar,
+.onboarding-modal__paginator,
+.error-modal__footer {
   background: darken($ui-base-color, 6%);
+
+  .onboarding-modal__nav,
+  .error-modal__nav {
+    &:hover,
+    &:focus,
+    &:active {
+      background-color: darken($ui-base-color, 12%);
+    }
+  }
+}
+
+.display-case__case {
+  background: $white;
+}
+
+.embed-modal .embed-modal__container .embed-modal__html {
+  background: $white;
+
+  &:focus {
+    background: darken($ui-base-color, 6%);
+  }
 }
 
 .react-toggle-track {
   background: $ui-secondary-color;
 }
 
+.react-toggle:hover:not(.react-toggle--disabled) .react-toggle-track {
+  background: darken($ui-secondary-color, 10%);
+}
+
+.react-toggle.react-toggle--checked:hover:not(.react-toggle--disabled) .react-toggle-track {
+  background: lighten($ui-highlight-color, 10%);
+}
+
 // Change the default color used for the text in an empty column or on the error column
 .empty-column-indicator,
 .error-column {

app/javascript/styles/mastodon/components.scss

@@ -3284,6 +3284,15 @@ a.status-card {
 }
 
 .search__icon {
+  &::-moz-focus-inner {
+    border: 0;
+  }
+
+  &::-moz-focus-inner,
+  &:focus {
+    outline: 0 !important;
+  }
+
   .fa {
     position: absolute;
     top: 10px;
@@ -3333,40 +3342,33 @@ a.status-card {
 .search-results__header {
   color: $dark-text-color;
   background: lighten($ui-base-color, 2%);
-  border-bottom: 1px solid darken($ui-base-color, 4%);
-  padding: 15px 10px;
-  font-size: 14px;
+  padding: 15px;
   font-weight: 500;
+  font-size: 16px;
+  cursor: default;
+
+  .fa {
+    display: inline-block;
+    margin-right: 5px;
+  }
 }
 
 .search-results__section {
-  margin-bottom: 20px;
+  margin-bottom: 5px;
 
   h5 {
-    position: relative;
-
-    &::before {
-      content: "";
-      display: block;
-      position: absolute;
-      left: 0;
-      right: 0;
-      top: 50%;
-      width: 100%;
-      height: 0;
-      border-top: 1px solid lighten($ui-base-color, 8%);
-    }
+    background: darken($ui-base-color, 4%);
+    border-bottom: 1px solid lighten($ui-base-color, 8%);
+    cursor: default;
+    display: flex;
+    padding: 15px;
+    font-weight: 500;
+    font-size: 16px;
+    color: $dark-text-color;
 
-    span {
+    .fa {
       display: inline-block;
-      background: $ui-base-color;
-      color: $darker-text-color;
-      font-size: 14px;
-      font-weight: 500;
-      padding: 10px;
-      position: relative;
-      z-index: 1;
-      cursor: default;
+      margin-right: 5px;
     }
   }
 
@@ -5209,3 +5211,81 @@ noscript {
     background: $ui-base-color;
   }
 }
+
+.trends {
+  &__header {
+    color: $dark-text-color;
+    background: lighten($ui-base-color, 2%);
+    border-bottom: 1px solid darken($ui-base-color, 4%);
+    font-weight: 500;
+    padding: 15px;
+    font-size: 16px;
+    cursor: default;
+
+    .fa {
+      display: inline-block;
+      margin-right: 5px;
+    }
+  }
+
+  &__item {
+    display: flex;
+    align-items: center;
+    padding: 15px;
+    border-bottom: 1px solid lighten($ui-base-color, 8%);
+
+    &:last-child {
+      border-bottom: 0;
+    }
+
+    &__name {
+      flex: 1 1 auto;
+      color: $dark-text-color;
+      overflow: hidden;
+      text-overflow: ellipsis;
+      white-space: nowrap;
+
+      strong {
+        font-weight: 500;
+      }
+
+      a {
+        color: $darker-text-color;
+        text-decoration: none;
+        font-size: 14px;
+        font-weight: 500;
+        display: block;
+        overflow: hidden;
+        text-overflow: ellipsis;
+        white-space: nowrap;
+
+        &:hover,
+        &:focus,
+        &:active {
+          span {
+            text-decoration: underline;
+          }
+        }
+      }
+    }
+
+    &__current {
+      flex: 0 0 auto;
+      width: 100px;
+      font-size: 24px;
+      line-height: 36px;
+      font-weight: 500;
+      text-align: center;
+      color: $secondary-text-color;
+    }
+
+    &__sparkline {
+      flex: 0 0 auto;
+      width: 50px;
+
+      path {
+        stroke: lighten($highlight-text-color, 6%) !important;
+      }
+    }
+  }
+}

app/lib/activitypub/activity/create.rb

@@ -78,9 +78,12 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     return if tag['name'].blank?
 
     hashtag = tag['name'].gsub(/\A#/, '').mb_chars.downcase
-    hashtag = Tag.where(name: hashtag).first_or_initialize(name: hashtag)
+    hashtag = Tag.where(name: hashtag).first_or_create(name: hashtag)
 
-    status.tags << hashtag unless status.tags.include?(hashtag)
+    return if status.tags.include?(hashtag)
+
+    status.tags << hashtag
+    TrendingTags.record_use!(hashtag, status.account, status.created_at)
   rescue ActiveRecord::RecordInvalid
     nil
   end

app/lib/ostatus/atom_serializer.rb

@@ -354,7 +354,7 @@ class OStatus::AtomSerializer
     append_element(entry, 'summary', status.spoiler_text, 'xml:lang': status.language) if status.spoiler_text?
     append_element(entry, 'content', Formatter.instance.format(status).to_str, type: 'html', 'xml:lang': status.language)
 
-    status.mentions.order(:id).each do |mentioned|
+    status.mentions.sort_by(&:id).each do |mentioned|
       append_element(entry, 'link', nil, rel: :mentioned, 'ostatus:object-type': OStatus::TagManager::TYPES[:person], href: OStatus::TagManager.instance.uri_for(mentioned.account))
     end
 

app/models/concerns/remotable.rb

@@ -41,7 +41,7 @@ module Remotable
         rescue HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError, Paperclip::Errors::NotIdentifiedByImageMagickError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e
           Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}"
           nil
-        rescue Paperclip::Error => e
+        rescue Paperclip::Error, Mastodon::DimensionsValidationError => e
           Rails.logger.debug "Error processing remote #{attachment_name}: #{e}"
           nil
         end

app/models/status.rb

@@ -195,12 +195,45 @@ class Status < ApplicationRecord
       where(account: [account] + account.following).where(visibility: [:public, :unlisted, :private])
     end
 
-    def as_direct_timeline(account)
-      query = joins("LEFT OUTER JOIN mentions ON statuses.id = mentions.status_id AND mentions.account_id = #{account.id}")
-              .where("mentions.account_id = #{account.id} OR statuses.account_id = #{account.id}")
-              .where(visibility: [:direct])
-
-      apply_timeline_filters(query, account, false)
+    def as_direct_timeline(account, limit = 20, max_id = nil, since_id = nil, cache_ids = false)
+      # direct timeline is mix of direct message from_me and to_me.
+      # 2 querys are executed with pagination.
+      # constant expression using arel_table is required for partial index
+
+      # _from_me part does not require any timeline filters
+      query_from_me = where(account_id: account.id)
+                      .where(Status.arel_table[:visibility].eq(3))
+                      .limit(limit)
+                      .order('statuses.id DESC')
+
+      # _to_me part requires mute and block filter.
+      # FIXME: may we check mutes.hide_notifications?
+      query_to_me = Status
+                    .joins(:mentions)
+                    .merge(Mention.where(account_id: account.id))
+                    .where(Status.arel_table[:visibility].eq(3))
+                    .limit(limit)
+                    .order('mentions.status_id DESC')
+                    .not_excluded_by_account(account)
+
+      if max_id.present?
+        query_from_me = query_from_me.where('statuses.id < ?', max_id)
+        query_to_me = query_to_me.where('mentions.status_id < ?', max_id)
+      end
+
+      if since_id.present?
+        query_from_me = query_from_me.where('statuses.id > ?', since_id)
+        query_to_me = query_to_me.where('mentions.status_id > ?', since_id)
+      end
+
+      if cache_ids
+        # returns array of cache_ids object that have id and updated_at
+        (query_from_me.cache_ids.to_a + query_to_me.cache_ids.to_a).uniq(&:id).sort_by(&:id).reverse.take(limit)
+      else
+        # returns ActiveRecord.Relation
+        items = (query_from_me.select(:id).to_a + query_to_me.select(:id).to_a).uniq(&:id).sort_by(&:id).reverse.take(limit)
+        Status.where(id: items.map(&:id))
+      end
     end
 
     def as_public_timeline(account = nil, local_only = false)

app/models/tag.rb

@@ -21,6 +21,22 @@ class Tag < ApplicationRecord
     name
   end
 
+  def history
+    days = []
+
+    7.times do |i|
+      day = i.days.ago.beginning_of_day.to_i
+
+      days << {
+        day: day.to_s,
+        uses: Redis.current.get("activity:tags:#{id}:#{day}") || '0',
+        accounts: Redis.current.pfcount("activity:tags:#{id}:#{day}:accounts").to_s,
+      }
+    end
+
+    days
+  end
+
   class << self
     def search_for(term, limit = 5)
       pattern = sanitize_sql_like(term.strip) + '%'

app/models/trending_tags.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+class TrendingTags
+  KEY                  = 'trending_tags'
+  HALF_LIFE            = 1.day.to_i
+  MAX_ITEMS            = 500
+  EXPIRE_HISTORY_AFTER = 7.days.seconds
+
+  class << self
+    def record_use!(tag, account, at_time = Time.now.utc)
+      return if disallowed_hashtags.include?(tag.name) || account.silenced?
+
+      increment_vote!(tag.id, at_time)
+      increment_historical_use!(tag.id, at_time)
+      increment_unique_use!(tag.id, account.id, at_time)
+    end
+
+    def get(limit)
+      tag_ids = redis.zrevrange(KEY, 0, limit).map(&:to_i)
+      tags    = Tag.where(id: tag_ids).to_a.map { |tag| [tag.id, tag] }.to_h
+      tag_ids.map { |tag_id| tags[tag_id] }.compact
+    end
+
+    private
+
+    def increment_vote!(tag_id, at_time)
+      redis.zincrby(KEY, (2**((at_time.to_i - epoch) / HALF_LIFE)).to_f, tag_id.to_s)
+      redis.zremrangebyrank(KEY, 0, -MAX_ITEMS) if rand < (2.to_f / MAX_ITEMS)
+    end
+
+    def increment_historical_use!(tag_id, at_time)
+      key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}"
+      redis.incrby(key, 1)
+      redis.expire(key, EXPIRE_HISTORY_AFTER)
+    end
+
+    def increment_unique_use!(tag_id, account_id, at_time)
+      key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts"
+      redis.pfadd(key, account_id)
+      redis.expire(key, EXPIRE_HISTORY_AFTER)
+    end
+
+    # The epoch needs to be 2.5 years in the future if the half-life is one day
+    # While dynamic, it will always be the same within one year
+    def epoch
+      @epoch ||= Date.new(Date.current.year + 2.5, 10, 1).to_datetime.to_i
+    end
+
+    def disallowed_hashtags
+      return @disallowed_hashtags if defined?(@disallowed_hashtags)
+
+      @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags
+      @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String
+      @disallowed_hashtags = @disallowed_hashtags.map(&:downcase)
+    end
+
+    def redis
+      Redis.current
+    end
+  end
+end

app/serializers/rest/tag_serializer.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class REST::TagSerializer < ActiveModel::Serializer
+  include RoutingHelper
+
+  attributes :name, :url, :history
+
+  def url
+    tag_url(object)
+  end
+end

app/serializers/rest/v2/search_serializer.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class REST::V2::SearchSerializer < ActiveModel::Serializer
+  has_many :accounts, serializer: REST::AccountSerializer
+  has_many :statuses, serializer: REST::StatusSerializer
+  has_many :hashtags, serializer: REST::TagSerializer
+end

app/services/process_hashtags_service.rb

@@ -4,8 +4,10 @@ class ProcessHashtagsService < BaseService
   def call(status, tags = [])
     tags = Extractor.extract_hashtags(status.text) if status.local?
 
-    tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |tag|
-      status.tags << Tag.where(name: tag).first_or_initialize(name: tag)
+    tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |name|
+      tag = Tag.where(name: name).first_or_create(name: name)
+      status.tags << tag
+      TrendingTags.record_use!(tag, status.account, status.created_at)
     end
   end
 end

app/validators/email_mx_validator.rb

@@ -4,7 +4,7 @@ require 'resolv'
 
 class EmailMxValidator < ActiveModel::Validator
   def validate(user)
-    return if Rails.env.test?
+    return if Rails.env.test? || Rails.env.development?
     user.errors.add(:email, I18n.t('users.invalid_email')) if invalid_mx?(user.email)
   end
 

config/routes.rb

@@ -271,6 +271,7 @@ Rails.application.routes.draw do
       resources :favourites, only: [:index]
       resources :bookmarks,  only: [:index]
       resources :reports,    only: [:index, :create]
+      resources :trends,     only: [:index]
 
       namespace :apps do
         get :verify_credentials, to: 'credentials#show'
@@ -332,6 +333,10 @@ Rails.application.routes.draw do
       end
     end
 
+    namespace :v2 do
+      get '/search', to: 'search#index', as: :search
+    end
+
     namespace :web do
       resource :settings, only: [:update]
       resource :embed, only: [:create]

config/webpack/production.js

@@ -6,8 +6,9 @@ const CompressionPlugin = require('compression-webpack-plugin');
 const sharedConfig = require('./shared.js');
 const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin;
 const OfflinePlugin = require('offline-plugin');
-const { env, publicPath } = require('./configuration.js');
+const { publicPath } = require('./configuration.js');
 const path = require('path');
+const { URL } = require('whatwg-url');
 
 let compressionAlgorithm;
 try {
@@ -19,6 +20,21 @@ try {
   compressionAlgorithm = 'gzip';
 }
 
+let attachmentHost;
+
+if (process.env.S3_ENABLED === 'true') {
+  if (process.env.S3_CLOUDFRONT_HOST) {
+    attachmentHost = process.env.S3_CLOUDFRONT_HOST;
+  } else {
+    attachmentHost = process.env.S3_HOSTNAME || `s3-${process.env.S3_REGION || 'us-east-1'}.amazonaws.com`;
+  }
+} else if (process.env.SWIFT_ENABLED === 'true') {
+  const { host } = new URL(process.env.SWIFT_OBJECT_URL);
+  attachmentHost = host;
+} else {
+  attachmentHost = null;
+}
+
 module.exports = merge(sharedConfig, {
   output: {
     filename: '[name]-[chunkhash].js',
@@ -90,7 +106,7 @@ module.exports = merge(sharedConfig, {
         '**/*.woff',
       ],
       ServiceWorker: {
-        entry: `imports-loader?process.env=>${encodeURIComponent(JSON.stringify(env))}!${encodeURI(path.join(__dirname, '../../app/javascript/mastodon/service_worker/entry.js'))}`,
+        entry: `imports-loader?ATTACHMENT_HOST=>${encodeURIComponent(JSON.stringify(attachmentHost))}!${encodeURI(path.join(__dirname, '../../app/javascript/mastodon/service_worker/entry.js'))}`,
         cacheName: 'mastodon',
         output: '../assets/sw.js',
         publicPath: '/sw.js',

package.json

@@ -98,6 +98,7 @@
     "react-redux-loading-bar": "^2.9.3",
     "react-router-dom": "^4.1.1",
     "react-router-scroll-4": "^1.0.0-beta.1",
+    "react-sparklines": "^1.7.0",
     "react-swipeable-views": "^0.12.3",
     "react-textarea-autosize": "^5.2.1",
     "react-toggle": "^4.0.1",
@@ -121,7 +122,8 @@
     "webpack-bundle-analyzer": "^2.9.1",
     "webpack-manifest-plugin": "^1.2.1",
     "webpack-merge": "^4.1.1",
-    "websocket.js": "^0.1.12"
+    "websocket.js": "^0.1.12",
+    "whatwg-url": "^6.4.1"
   },
   "devDependencies": {
     "babel-eslint": "^8.2.1",

spec/controllers/emojis_controller_spec.rb

@@ -0,0 +1,17 @@
+require 'rails_helper'
+
+describe EmojisController do
+  render_views
+
+  let(:emoji) { Fabricate(:custom_emoji) }
+
+  describe 'GET #show' do
+    subject(:responce) { get :show, params: { id: emoji.id, format: :json } }
+    subject(:body) { JSON.parse(response.body, symbolize_names: true) }
+
+    it 'returns the right response' do
+      expect(responce).to have_http_status 200
+      expect(body[:name]).to eq ':coolcat:'
+    end
+  end
+end

spec/models/status_spec.rb

@@ -370,24 +370,25 @@ RSpec.describe Status, type: :model do
       expect(@results).to_not include(@followed_public_status)
     end
 
-    it 'includes direct statuses mentioning recipient from followed' do
-      Fabricate(:mention, account: account, status: @followed_direct_status)
-      expect(@results).to include(@followed_direct_status)
-    end
-
     it 'does not include direct statuses not mentioning recipient from followed' do
       expect(@results).to_not include(@followed_direct_status)
     end
 
-    it 'includes direct statuses mentioning recipient from non-followed' do
-      Fabricate(:mention, account: account, status: @not_followed_direct_status)
-      expect(@results).to include(@not_followed_direct_status)
-    end
-
     it 'does not include direct statuses not mentioning recipient from non-followed' do
       expect(@results).to_not include(@not_followed_direct_status)
     end
 
+    it 'includes direct statuses mentioning recipient from followed' do
+      Fabricate(:mention, account: account, status: @followed_direct_status)
+      results2 = Status.as_direct_timeline(account)
+      expect(results2).to include(@followed_direct_status)
+    end
+
+    it 'includes direct statuses mentioning recipient from non-followed' do
+      Fabricate(:mention, account: account, status: @not_followed_direct_status)
+      results2 = Status.as_direct_timeline(account)
+      expect(results2).to include(@not_followed_direct_status)
+    end
   end
 
   describe '.as_public_timeline' do

yarn.lock

@@ -4391,6 +4391,10 @@ lodash.restparam@^3.0.0:
   version "3.6.1"
   resolved "https://registry.yarnpkg.com/lodash.restparam/-/lodash.restparam-3.6.1.tgz#936a4e309ef330a7645ed4145986c85ae5b20805"
 
+lodash.sortby@^4.7.0:
+  version "4.7.0"
+  resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438"
+
 lodash.tail@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/lodash.tail/-/lodash.tail-4.1.1.tgz#d2333a36d9e7717c8ad2f7cacafec7c32b444664"
@@ -6146,6 +6150,12 @@ react-router@^4.2.0:
     prop-types "^15.5.4"
     warning "^3.0.0"
 
+react-sparklines@^1.7.0:
+  version "1.7.0"
+  resolved "https://registry.yarnpkg.com/react-sparklines/-/react-sparklines-1.7.0.tgz#9b1d97e8c8610095eeb2ad658d2e1fcf91f91a60"
+  dependencies:
+    prop-types "^15.5.10"
+
 react-swipeable-views-core@^0.12.11:
   version "0.12.11"
   resolved "https://registry.yarnpkg.com/react-swipeable-views-core/-/react-swipeable-views-core-0.12.11.tgz#3cf2b4daffbb36f9d69bd19bf5b2d5370b6b2c1b"
@@ -7273,6 +7283,12 @@ tough-cookie@^2.3.2, tough-cookie@~2.3.0, tough-cookie@~2.3.3:
   dependencies:
     punycode "^1.4.1"
 
+tr46@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/tr46/-/tr46-1.0.1.tgz#a8b13fd6bfd2489519674ccde55ba3693b706d09"
+  dependencies:
+    punycode "^2.1.0"
+
 tr46@~0.0.3:
   version "0.0.3"
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
@@ -7510,7 +7526,7 @@ webidl-conversions@^3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
 
-webidl-conversions@^4.0.0:
+webidl-conversions@^4.0.0, webidl-conversions@^4.0.2:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad"
 
@@ -7653,6 +7669,14 @@ whatwg-url@^4.3.0:
     tr46 "~0.0.3"
     webidl-conversions "^3.0.0"
 
+whatwg-url@^6.4.1:
+  version "6.4.1"
+  resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-6.4.1.tgz#fdb94b440fd4ad836202c16e9737d511f012fd67"
+  dependencies:
+    lodash.sortby "^4.7.0"
+    tr46 "^1.0.1"
+    webidl-conversions "^4.0.2"
+
 whet.extend@~0.9.9:
   version "0.9.9"
   resolved "https://registry.yarnpkg.com/whet.extend/-/whet.extend-0.9.9.tgz#f877d5bf648c97e5aa542fadc16d6a259b9c11a1"