diff --git a/app/javascript/mastodon/emoji.js b/app/javascript/mastodon/emoji.js index a41dfdd1d2..865b85b611 100644 --- a/app/javascript/mastodon/emoji.js +++ b/app/javascript/mastodon/emoji.js @@ -3,28 +3,48 @@ import Trie from 'substring-trie'; const trie = new Trie(Object.keys(unicodeMapping)); -const emojify = str => { - let rtn = ''; - for (;;) { - let match, i = 0; - while (i < str.length && str[i] !== '<' && !(match = trie.search(str.slice(i)))) { - i += str.codePointAt(i) < 65536 ? 1 : 2; - } - if (i === str.length) - break; - else if (str[i] === '<') { - let tagend = str.indexOf('>', i + 1) + 1; - if (!tagend) - break; - rtn += str.slice(0, tagend); - str = str.slice(tagend); - } else { - const [filename, shortCode] = unicodeMapping[match]; - rtn += str.slice(0, i) + ``; - str = str.slice(i + match.length); +const emojify = (str, customEmojis = {}) => { + // This walks through the string from start to end, ignoring any tags (
,
, etc.)
+ // and replacing valid unicode strings
+ // that _aren't_ within tags with an version.
+ // The goal is to be the same as an emojione.regUnicode replacement, but faster.
+ let i = -1;
+ let insideTag = false;
+ let insideShortname = false;
+ let shortnameStartIndex = -1;
+ let match;
+ while (++i < str.length) {
+ const char = str.charAt(i);
+ if (insideShortname && char === ':') {
+ const shortname = str.substring(shortnameStartIndex, i + 1);
+ if (shortname in customEmojis) {
+ const replacement = ``;
+ str = str.substring(0, shortnameStartIndex) + replacement + str.substring(i + 1);
+ i += (replacement.length - shortname.length - 1); // jump ahead the length we've added to the string
+ } else {
+ i--;
+ }
+ insideShortname = false;
+ } else if (insideTag && char === '>') {
+ insideTag = false;
+ } else if (char === '<') {
+ insideTag = true;
+ insideShortname = false;
+ } else if (!insideTag && char === ':') {
+ insideShortname = true;
+ shortnameStartIndex = i;
+ } else if (!insideTag && (match = trie.search(str.substring(i)))) {
+ const unicodeStr = match;
+ if (unicodeStr in unicodeMapping) {
+ const [filename, shortCode] = unicodeMapping[unicodeStr];
+ const alt = unicodeStr;
+ const replacement = ``;
+ str = str.substring(0, i) + replacement + str.substring(i + unicodeStr.length);
+ i += (replacement.length - unicodeStr.length); // jump ahead the length we've added to the string
+ }
}
}
- return rtn + str;
+ return str;
};
export default emojify;
diff --git a/app/javascript/mastodon/reducers/statuses.js b/app/javascript/mastodon/reducers/statuses.js
index 7f906bef61..38b23504ed 100644
--- a/app/javascript/mastodon/reducers/statuses.js
+++ b/app/javascript/mastodon/reducers/statuses.js
@@ -58,9 +58,14 @@ const normalizeStatus = (state, status) => {
}
const searchContent = [status.spoiler_text, status.content].join(' ').replace(/
/g, '\n').replace(/<\/p>
/g, '\n\n');
+ const emojiMap = normalStatus.emojis.reduce((obj, emoji) => {
+ obj[`:${emoji.shortcode}:`] = emoji.url;
+ return obj;
+ }, {});
+
normalStatus.search_index = domParser.parseFromString(searchContent, 'text/html').documentElement.textContent;
- normalStatus.contentHtml = emojify(normalStatus.content);
- normalStatus.spoilerHtml = emojify(escapeTextContentForBrowser(normalStatus.spoiler_text || ''));
+ normalStatus.contentHtml = emojify(normalStatus.content, emojiMap);
+ normalStatus.spoilerHtml = emojify(escapeTextContentForBrowser(normalStatus.spoiler_text || ''), emojiMap);
return state.update(status.id, ImmutableMap(), map => map.mergeDeep(fromJS(normalStatus)));
};
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 894759d9a4..41f2b0bad7 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -61,6 +61,8 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
process_hashtag tag, status
when 'Mention'
process_mention tag, status
+ when 'Emoji'
+ process_emoji tag, status
end
end
end
@@ -79,6 +81,17 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
account.mentions.create(status: status)
end
+ def process_emoji(tag, _status)
+ shortcode = tag['name'].delete(':')
+ emoji = CustomEmoji.find_by(shortcode: shortcode, domain: @account.domain)
+
+ return if !emoji.nil? || skip_download?
+
+ emoji = CustomEmoji.new(domain: @account.domain, shortcode: shortcode)
+ emoji.image_remote_url = tag['href']
+ emoji.save
+ end
+
def process_attachments(status)
return unless @object['attachment'].is_a?(Array)
diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb
index 575830190d..29fea27de7 100644
--- a/app/lib/formatter.rb
+++ b/app/lib/formatter.rb
@@ -9,7 +9,7 @@ class Formatter
include ActionView::Helpers::TextHelper
- def format(status)
+ def format(status, options = {})
if status.reblog?
prepend_reblog = status.reblog.account.acct
status = status.proper
@@ -19,7 +19,11 @@ class Formatter
raw_content = status.text
- return reformat(raw_content) unless status.local?
+ unless status.local?
+ html = reformat(raw_content)
+ html = encode_custom_emojis(html, status.emojis) if options[:custom_emojify]
+ return html
+ end
linkable_accounts = status.mentions.map(&:account)
linkable_accounts << status.account
@@ -27,6 +31,7 @@ class Formatter
html = raw_content
html = "RT @#{prepend_reblog} #{html}" if prepend_reblog
html = encode_and_link_urls(html, linkable_accounts)
+ html = encode_custom_emojis(html, status.emojis) if options[:custom_emojify]
html = simple_format(html, {}, sanitize: false)
html = html.delete("\n")
@@ -39,7 +44,9 @@ class Formatter
def plaintext(status)
return status.text if status.local?
- strip_tags(status.text)
+
+ text = status.text.gsub(/(
|
|<\/p>)+/) { |match| "#{match}\n" }
+ strip_tags(text)
end
def simplified_format(account)
@@ -76,6 +83,47 @@ class Formatter
end
end
+ def encode_custom_emojis(html, emojis)
+ return html if emojis.empty?
+
+ emoji_map = emojis.map { |e| [e.shortcode, full_asset_url(e.image.url)] }.to_h
+
+ i = -1
+ inside_tag = false
+ inside_shortname = false
+ shortname_start_index = -1
+
+ while i + 1 < html.size
+ i += 1
+
+ if inside_shortname && html[i] == ':'
+ shortcode = html[shortname_start_index + 1..i - 1]
+ emoji = emoji_map[shortcode]
+
+ if emoji
+ replacement = ""
+ before_html = shortname_start_index.positive? ? html[0..shortname_start_index - 1] : ''
+ html = before_html + replacement + html[i + 1..-1]
+ i += replacement.size - (shortcode.size + 2) - 1
+ else
+ i -= 1
+ end
+
+ inside_shortname = false
+ elsif inside_tag && html[i] == '>'
+ inside_tag = false
+ elsif html[i] == '<'
+ inside_tag = true
+ inside_shortname = false
+ elsif !inside_tag && html[i] == ':'
+ inside_shortname = true
+ shortname_start_index = i
+ end
+ end
+
+ html
+ end
+
def rewrite(text, entities)
chars = text.to_s.to_char_a
diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb
index 1a23c9efa3..d3f1629c45 100644
--- a/app/lib/ostatus/activity/creation.rb
+++ b/app/lib/ostatus/activity/creation.rb
@@ -42,6 +42,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
save_mentions(status)
save_hashtags(status)
save_media(status)
+ save_emojis(status)
end
if thread? && status.thread.nil?
@@ -150,6 +151,25 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
end
end
+ def save_emojis(parent)
+ do_not_download = DomainBlock.find_by(domain: parent.account.domain)&.reject_media?
+
+ return if do_not_download
+
+ @xml.xpath('./xmlns:link[@rel="emoji"]', xmlns: TagManager::XMLNS).each do |link|
+ next unless link['href'] && link['name']
+
+ shortcode = link['name'].delete(':')
+ emoji = CustomEmoji.find_by(shortcode: shortcode, domain: parent.account.domain)
+
+ next unless emoji.nil?
+
+ emoji = CustomEmoji.new(shortcode: shortcode, domain: parent.account.domain)
+ emoji.image_remote_url = link['href']
+ emoji.save
+ end
+ end
+
def account_from_href(href)
url = Addressable::URI.parse(href).normalize
diff --git a/app/lib/ostatus/atom_serializer.rb b/app/lib/ostatus/atom_serializer.rb
index b8e22a3813..a6a5cb0c45 100644
--- a/app/lib/ostatus/atom_serializer.rb
+++ b/app/lib/ostatus/atom_serializer.rb
@@ -368,5 +368,9 @@ class OStatus::AtomSerializer
end
append_element(entry, 'mastodon:scope', status.visibility)
+
+ status.emojis.each do |emoji|
+ append_element(entry, 'link', nil, rel: :emoji, href: full_asset_url(emoji.image.url), name: emoji.shortcode)
+ end
end
end
diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb
new file mode 100644
index 0000000000..f4d3b16a03
--- /dev/null
+++ b/app/models/custom_emoji.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: custom_emojis
+#
+# id :integer not null, primary key
+# shortcode :string default(""), not null
+# domain :string
+# image_file_name :string
+# image_content_type :string
+# image_file_size :integer
+# image_updated_at :datetime
+# created_at :datetime not null
+# updated_at :datetime not null
+#
+
+class CustomEmoji < ApplicationRecord
+ SHORTCODE_RE_FRAGMENT = '[a-zA-Z0-9_]{2,}'
+
+ SCAN_RE = /(?<=[^[:alnum:]:]|\n|^)
+ :(#{SHORTCODE_RE_FRAGMENT}):
+ (?=[^[:alnum:]:]|$)/x
+
+ has_attached_file :image
+
+ validates_attachment :image, content_type: { content_type: 'image/png' }, presence: true, size: { in: 0..50.kilobytes }
+ validates :shortcode, uniqueness: { scope: :domain }, format: { with: /\A#{SHORTCODE_RE_FRAGMENT}\z/ }, length: { minimum: 2 }
+
+ include Remotable
+
+ class << self
+ def from_text(text, domain)
+ return [] if text.blank?
+ shortcodes = text.scan(SCAN_RE).map(&:first)
+ where(shortcode: shortcodes, domain: domain)
+ end
+ end
+end
diff --git a/app/models/status.rb b/app/models/status.rb
index 2a2cdcf6ea..326d128d6d 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -131,6 +131,10 @@ class Status < ApplicationRecord
!sensitive? && media_attachments.any?
end
+ def emojis
+ CustomEmoji.from_text(text, account.domain)
+ end
+
after_create :store_uri, if: :local?
before_validation :prepare_contents, if: :local?
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 166214eee5..e5d8e3f030 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -57,7 +57,7 @@ class ActivityPub::NoteSerializer < ActiveModel::Serializer
end
def virtual_tags
- object.mentions + object.tags
+ object.mentions + object.tags + object.emojis
end
def atom_uri
@@ -137,4 +137,22 @@ class ActivityPub::NoteSerializer < ActiveModel::Serializer
"##{object.name}"
end
end
+
+ class CustomEmojiSerializer < ActiveModel::Serializer
+ include RoutingHelper
+
+ attributes :type, :href, :name
+
+ def type
+ 'Emoji'
+ end
+
+ def href
+ full_asset_url(object.image.url)
+ end
+
+ def name
+ ":#{object.shortcode}:"
+ end
+ end
end
diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb
index 298a3bb40f..d8efa8e60b 100644
--- a/app/serializers/rest/status_serializer.rb
+++ b/app/serializers/rest/status_serializer.rb
@@ -17,6 +17,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
has_many :media_attachments, serializer: REST::MediaAttachmentSerializer
has_many :mentions
has_many :tags
+ has_many :emojis
def current_user?
!current_user.nil?
@@ -106,4 +107,14 @@ class REST::StatusSerializer < ActiveModel::Serializer
tag_url(object)
end
end
+
+ class CustomEmojiSerializer < ActiveModel::Serializer
+ include RoutingHelper
+
+ attributes :shortcode, :url
+
+ def url
+ full_asset_url(object.image.url)
+ end
+ end
end
diff --git a/app/views/stream_entries/_detailed_status.html.haml b/app/views/stream_entries/_detailed_status.html.haml
index dd94562609..692d5a6d5c 100644
--- a/app/views/stream_entries/_detailed_status.html.haml
+++ b/app/views/stream_entries/_detailed_status.html.haml
@@ -17,7 +17,7 @@
%p{ style: 'margin-bottom: 0' }<
%span.p-summary> #{status.spoiler_text}
%a.status__content__spoiler-link{ href: '#' }= t('statuses.show_more')
- .e-content{ lang: status.language, style: "display: #{status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }= Formatter.instance.format(status)
+ .e-content{ lang: status.language, style: "display: #{status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }= Formatter.instance.format(status, custom_emojify: true)
- if !status.media_attachments.empty?
- if status.media_attachments.first.video?
diff --git a/app/views/stream_entries/_simple_status.html.haml b/app/views/stream_entries/_simple_status.html.haml
index 55aa97f323..f9a530d38a 100644
--- a/app/views/stream_entries/_simple_status.html.haml
+++ b/app/views/stream_entries/_simple_status.html.haml
@@ -18,7 +18,7 @@
%p{ style: 'margin-bottom: 0' }<
%span.p-summary> #{status.spoiler_text}
%a.status__content__spoiler-link{ href: '#' }= t('statuses.show_more')
- .e-content{ lang: status.language, style: "display: #{status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }= Formatter.instance.format(status)
+ .e-content{ lang: status.language, style: "display: #{status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }= Formatter.instance.format(status, custom_emojify: true)
- unless status.media_attachments.empty?
- if status.media_attachments.first.video?
diff --git a/db/migrate/20170917153509_create_custom_emojis.rb b/db/migrate/20170917153509_create_custom_emojis.rb
new file mode 100644
index 0000000000..4040c83125
--- /dev/null
+++ b/db/migrate/20170917153509_create_custom_emojis.rb
@@ -0,0 +1,13 @@
+class CreateCustomEmojis < ActiveRecord::Migration[5.1]
+ def change
+ create_table :custom_emojis do |t|
+ t.string :shortcode, null: false, default: ''
+ t.string :domain
+ t.attachment :image
+
+ t.timestamps
+ end
+
+ add_index :custom_emojis, [:shortcode, :domain], unique: true
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index f2ca2af696..9f42d46dd6 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20170913000752) do
+ActiveRecord::Schema.define(version: 20170917153509) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@@ -89,6 +89,18 @@ ActiveRecord::Schema.define(version: 20170913000752) do
t.index ["uri"], name: "index_conversations_on_uri", unique: true
end
+ create_table "custom_emojis", force: :cascade do |t|
+ t.string "shortcode", default: "", null: false
+ t.string "domain"
+ t.string "image_file_name"
+ t.string "image_content_type"
+ t.integer "image_file_size"
+ t.datetime "image_updated_at"
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ t.index ["shortcode", "domain"], name: "index_custom_emojis_on_shortcode_and_domain", unique: true
+ end
+
create_table "domain_blocks", id: :serial, force: :cascade do |t|
t.string "domain", default: "", null: false
t.datetime "created_at", null: false
diff --git a/spec/fabricators/custom_emoji_fabricator.rb b/spec/fabricators/custom_emoji_fabricator.rb
new file mode 100644
index 0000000000..18a7d23dc4
--- /dev/null
+++ b/spec/fabricators/custom_emoji_fabricator.rb
@@ -0,0 +1,5 @@
+Fabricator(:custom_emoji) do
+ shortcode 'coolcat'
+ domain nil
+ image { File.open(Rails.root.join('spec', 'fixtures', 'files', 'emojo.png')) }
+end
diff --git a/spec/fixtures/files/emojo.png b/spec/fixtures/files/emojo.png
new file mode 100644
index 0000000000..cb5993499f
Binary files /dev/null and b/spec/fixtures/files/emojo.png differ
diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb
index fcb044ebcb..1a9520f04a 100644
--- a/spec/lib/activitypub/activity/create_spec.rb
+++ b/spec/lib/activitypub/activity/create_spec.rb
@@ -17,6 +17,7 @@ RSpec.describe ActivityPub::Activity::Create do
before do
stub_request(:get, 'http://example.com/attachment.png').to_return(request_fixture('avatar.txt'))
+ stub_request(:get, 'http://example.com/emoji.png').to_return(body: attachment_fixture('emojo.png'))
end
describe '#perform' do
@@ -217,5 +218,29 @@ RSpec.describe ActivityPub::Activity::Create do
expect(status.tags.map(&:name)).to include('test')
end
end
+
+ context 'with emojis' do
+ let(:object_json) do
+ {
+ id: 'bar',
+ type: 'Note',
+ content: 'Lorem ipsum :tinking:',
+ tag: [
+ {
+ type: 'Emoji',
+ href: 'http://example.com/emoji.png',
+ name: 'tinking',
+ },
+ ],
+ }
+ end
+
+ it 'creates status' do
+ status = sender.statuses.first
+
+ expect(status).to_not be_nil
+ expect(status.emojis.map(&:shortcode)).to include('tinking')
+ end
+ end
end
end
diff --git a/spec/lib/formatter_spec.rb b/spec/lib/formatter_spec.rb
index b714b317a3..71b6b78d2f 100644
--- a/spec/lib/formatter_spec.rb
+++ b/spec/lib/formatter_spec.rb
@@ -223,6 +223,45 @@ RSpec.describe Formatter do
include_examples 'encode and link URLs'
end
+
+ context 'with custom_emojify option' do
+ let!(:emoji) { Fabricate(:custom_emoji) }
+ let(:status) { Fabricate(:status, account: local_account, text: text) }
+
+ subject { Formatter.instance.format(status, custom_emojify: true) }
+
+ context 'with emoji at the start' do
+ let(:text) { ':coolcat: Beep boop' }
+
+ it 'converts shortcode to image tag' do
+ is_expected.to match(/
:coolcat: Beep boop
' }
+
+ it 'converts shortcode to image tag' do
+ is_expected.to match(/
Beep :coolcat: boop
' } + + it 'converts shortcode to image tag' do + is_expected.to match(/Beep :coolcat::coolcat:' } + + it 'does not touch the shortcodes' do + is_expected.to match(/:coolcat::coolcat:<\/p>/) + end + end + + context 'with emoji at the end' do + let(:text) { '
Beep boop
:coolcat: