Replace `Status#translatable?` with language matrix in separate endpoint (#24037)

th-downstream
Christian Schmidt 2 years ago committed by GitHub
parent f0734513f4
commit 7583523bf0

@ -484,7 +484,6 @@ RSpec/DescribedClass:
- 'spec/models/user_spec.rb'
- 'spec/policies/account_moderation_note_policy_spec.rb'
- 'spec/presenters/account_relationships_presenter_spec.rb'
- 'spec/presenters/instance_presenter_spec.rb'
- 'spec/presenters/status_relationships_presenter_spec.rb'
- 'spec/serializers/activitypub/note_serializer_spec.rb'
- 'spec/serializers/activitypub/update_poll_serializer_spec.rb'

@ -0,0 +1,23 @@
# frozen_string_literal: true
class Api::V1::Instances::TranslationLanguagesController < Api::BaseController
skip_before_action :require_authenticated_user!, unless: :whitelist_mode?
before_action :set_languages
def show
expires_in 1.day, public: true
render json: @languages
end
private
def set_languages
if TranslationService.configured?
@languages = Rails.cache.fetch('translation_service/languages', expires_in: 7.days, race_condition_ttl: 1.hour) { TranslationService.configured.languages }
@languages['und'] = @languages.delete(nil) if @languages.key?(nil)
else
@languages = {}
end
end
end

@ -5,6 +5,10 @@ export const SERVER_FETCH_REQUEST = 'Server_FETCH_REQUEST';
export const SERVER_FETCH_SUCCESS = 'Server_FETCH_SUCCESS';
export const SERVER_FETCH_FAIL = 'Server_FETCH_FAIL';
export const SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST = 'SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST';
export const SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS = 'SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS';
export const SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL = 'SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL';
export const EXTENDED_DESCRIPTION_REQUEST = 'EXTENDED_DESCRIPTION_REQUEST';
export const EXTENDED_DESCRIPTION_SUCCESS = 'EXTENDED_DESCRIPTION_SUCCESS';
export const EXTENDED_DESCRIPTION_FAIL = 'EXTENDED_DESCRIPTION_FAIL';
@ -37,6 +41,29 @@ const fetchServerFail = error => ({
error,
});
export const fetchServerTranslationLanguages = () => (dispatch, getState) => {
dispatch(fetchServerTranslationLanguagesRequest());
api(getState)
.get('/api/v1/instance/translation_languages').then(({ data }) => {
dispatch(fetchServerTranslationLanguagesSuccess(data));
}).catch(err => dispatch(fetchServerTranslationLanguagesFail(err)));
};
const fetchServerTranslationLanguagesRequest = () => ({
type: SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST,
});
const fetchServerTranslationLanguagesSuccess = translationLanguages => ({
type: SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS,
translationLanguages,
});
const fetchServerTranslationLanguagesFail = error => ({
type: SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL,
error,
});
export const fetchExtendedDescription = () => (dispatch, getState) => {
dispatch(fetchExtendedDescriptionRequest());

@ -3,6 +3,7 @@ import ImmutablePropTypes from 'react-immutable-proptypes';
import PropTypes from 'prop-types';
import { FormattedMessage, injectIntl } from 'react-intl';
import { Link } from 'react-router-dom';
import { connect } from 'react-redux';
import classnames from 'classnames';
import PollContainer from 'mastodon/containers/poll_container';
import Icon from 'mastodon/components/icon';
@ -47,7 +48,12 @@ class TranslateButton extends React.PureComponent {
}
export default @injectIntl
const mapStateToProps = state => ({
languages: state.getIn(['server', 'translationLanguages', 'items']),
});
export default @connect(mapStateToProps)
@injectIntl
class StatusContent extends React.PureComponent {
static contextTypes = {
@ -63,6 +69,7 @@ class StatusContent extends React.PureComponent {
onClick: PropTypes.func,
collapsable: PropTypes.bool,
onCollapsedToggle: PropTypes.func,
languages: ImmutablePropTypes.map,
intl: PropTypes.object,
};
@ -220,7 +227,9 @@ class StatusContent extends React.PureComponent {
const hidden = this.props.onExpandedToggle ? !this.props.expanded : this.state.hidden;
const renderReadMore = this.props.onClick && status.get('collapsed');
const renderTranslate = this.props.onTranslate && status.get('translatable');
const contentLocale = intl.locale.replace(/[_-].*/, '');
const targetLanguages = this.props.languages?.get(status.get('language') || 'und');
const renderTranslate = this.props.onTranslate && this.context.identity.signedIn && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('contentHtml').length > 0 && targetLanguages?.includes(contentLocale);
const content = { __html: status.get('translation') ? status.getIn(['translation', 'content']) : status.get('contentHtml') };
const spoilerContent = { __html: status.get('spoilerHtml') };

@ -13,7 +13,7 @@ import { debounce } from 'lodash';
import { uploadCompose, resetCompose, changeComposeSpoilerness } from '../../actions/compose';
import { expandHomeTimeline } from '../../actions/timelines';
import { expandNotifications } from '../../actions/notifications';
import { fetchServer } from '../../actions/server';
import { fetchServer, fetchServerTranslationLanguages } from '../../actions/server';
import { clearHeight } from '../../actions/height_cache';
import { focusApp, unfocusApp, changeLayout } from 'mastodon/actions/app';
import { synchronouslySubmitMarkers, submitMarkers, fetchMarkers } from 'mastodon/actions/markers';
@ -399,6 +399,7 @@ class UI extends React.PureComponent {
this.props.dispatch(fetchMarkers());
this.props.dispatch(expandHomeTimeline());
this.props.dispatch(expandNotifications());
this.props.dispatch(fetchServerTranslationLanguages());
setTimeout(() => this.props.dispatch(fetchServer()), 3000);
}

@ -2,6 +2,9 @@ import {
SERVER_FETCH_REQUEST,
SERVER_FETCH_SUCCESS,
SERVER_FETCH_FAIL,
SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST,
SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS,
SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL,
EXTENDED_DESCRIPTION_REQUEST,
EXTENDED_DESCRIPTION_SUCCESS,
EXTENDED_DESCRIPTION_FAIL,
@ -35,6 +38,12 @@ export default function server(state = initialState, action) {
return state.set('server', fromJS(action.server)).setIn(['server', 'isLoading'], false);
case SERVER_FETCH_FAIL:
return state.setIn(['server', 'isLoading'], false);
case SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST:
return state.setIn(['translationLanguages', 'isLoading'], true);
case SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS:
return state.setIn(['translationLanguages', 'items'], fromJS(action.translationLanguages)).setIn(['translationLanguages', 'isLoading'], false);
case SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL:
return state.setIn(['translationLanguages', 'isLoading'], false);
case EXTENDED_DESCRIPTION_REQUEST:
return state.setIn(['extendedDescription', 'isLoading'], true);
case EXTENDED_DESCRIPTION_SUCCESS:

@ -21,8 +21,8 @@ class TranslationService
ENV['DEEPL_API_KEY'].present? || ENV['LIBRE_TRANSLATE_ENDPOINT'].present?
end
def supported?(_source_language, _target_language)
false
def languages
{}
end
def translate(_text, _source_language, _target_language)

@ -17,25 +17,31 @@ class TranslationService::DeepL < TranslationService
end
end
def supported?(source_language, target_language)
source_language.in?(languages('source')) && target_language.in?(languages('target'))
def languages
source_languages = [nil] + fetch_languages('source')
# In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so
# they are supported but not returned by the API.
target_languages = %w(en pt) + fetch_languages('target')
source_languages.index_with { |language| target_languages.without(nil, language) }
end
private
def languages(type)
Rails.cache.fetch("translation_service/deepl/languages/#{type}", expires_in: 7.days, race_condition_ttl: 1.minute) do
request(:get, "/v2/languages?type=#{type}") do |res|
# In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so
# they are supported but not returned by the API.
extra = type == 'source' ? [nil] : %w(en pt)
languages = Oj.load(res.body_with_limit).map { |language| language['language'].downcase }
languages + extra
end
def fetch_languages(type)
request(:get, "/v2/languages?type=#{type}") do |res|
Oj.load(res.body_with_limit).map { |language| normalize_language(language['language']) }
end
end
def normalize_language(language)
subtags = language.split(/[_-]/)
subtags[0].downcase!
subtags[1]&.upcase!
subtags.join('-')
end
def request(verb, path, **options)
req = Request.new(verb, "#{base_url}#{path}", **options)
req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}")

@ -15,22 +15,18 @@ class TranslationService::LibreTranslate < TranslationService
end
end
def supported?(source_language, target_language)
languages.key?(source_language) && languages[source_language].include?(target_language)
end
private
def languages
Rails.cache.fetch('translation_service/libre_translate/languages', expires_in: 7.days, race_condition_ttl: 1.minute) do
request(:get, '/languages') do |res|
languages = Oj.load(res.body_with_limit).to_h { |language| [language['code'], language['targets']] }
languages[nil] = languages.values.flatten.uniq
languages
request(:get, '/languages') do |res|
languages = Oj.load(res.body_with_limit).to_h do |language|
[language['code'], language['targets'].without(language['code'])]
end
languages[nil] = languages.values.flatten.uniq.sort
languages
end
end
private
def request(verb, path, **options)
req = Request.new(verb, "#{@base_url}#{path}", allow_local: true, **options)
req.add_headers('Content-Type': 'application/json')

@ -232,16 +232,6 @@ class Status < ApplicationRecord
public_visibility? || unlisted_visibility?
end
def translatable?
translate_target_locale = I18n.locale.to_s.split(/[_-]/).first
distributable? &&
content.present? &&
language != translate_target_locale &&
TranslationService.configured? &&
TranslationService.configured.supported?(language, translate_target_locale)
end
alias sign? distributable?
def with_media?

@ -4,7 +4,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
include FormattingHelper
attributes :id, :created_at, :in_reply_to_id, :in_reply_to_account_id,
:sensitive, :spoiler_text, :visibility, :language, :translatable,
:sensitive, :spoiler_text, :visibility, :language,
:uri, :url, :replies_count, :reblogs_count,
:favourites_count, :edited_at
@ -50,10 +50,6 @@ class REST::StatusSerializer < ActiveModel::Serializer
object.account.user_shows_application? || (current_user? && current_user.account_id == object.account_id)
end
def translatable
current_user? && object.translatable?
end
def visibility
# This visibility is masked behind "private"
# to avoid API changes because there are no

@ -6,19 +6,29 @@ class TranslateStatusService < BaseService
include FormattingHelper
def call(status, target_language)
raise Mastodon::NotPermittedError unless status.translatable?
@status = status
@content = status_content_format(@status)
@target_language = target_language
raise Mastodon::NotPermittedError unless permitted?
Rails.cache.fetch("translations/#{@status.language}/#{@target_language}/#{content_hash}", expires_in: CACHE_TTL) { translation_backend.translate(@content, @status.language, @target_language) }
end
private
def translation_backend
TranslationService.configured
@translation_backend ||= TranslationService.configured
end
def permitted?
return false unless @status.distributable? && @status.content.present? && TranslationService.configured?
languages[@status.language]&.include?(@target_language)
end
def languages
Rails.cache.fetch('translation_service/languages', expires_in: 7.days, race_condition_ttl: 1.hour) { TranslationService.configured.languages }
end
def content_hash

@ -546,6 +546,7 @@ Rails.application.routes.draw do
resources :domain_blocks, only: [:index], controller: 'instances/domain_blocks'
resource :privacy_policy, only: [:show], controller: 'instances/privacy_policies'
resource :extended_description, only: [:show], controller: 'instances/extended_descriptions'
resource :translation_languages, only: [:show], controller: 'instances/translation_languages'
resource :activity, only: [:show], controller: 'instances/activity'
end

@ -0,0 +1,31 @@
# frozen_string_literal: true
require 'rails_helper'
describe Api::V1::Instances::TranslationLanguagesController do
describe 'GET #show' do
context 'when no translation service is configured' do
it 'returns empty language matrix' do
get :show
expect(response).to have_http_status(200)
expect(body_as_json).to eq({})
end
end
context 'when a translation service is configured' do
before do
service = instance_double(TranslationService::DeepL, languages: { nil => %w(en de), 'en' => ['de'] })
allow(TranslationService).to receive(:configured?).and_return(true)
allow(TranslationService).to receive(:configured).and_return(service)
end
it 'returns language matrix' do
get :show
expect(response).to have_http_status(200)
expect(body_as_json).to eq({ und: %w(en de), en: ['de'] })
end
end
end
end

@ -19,9 +19,10 @@ describe Api::V1::Statuses::TranslationsController do
before do
translation = TranslationService::Translation.new(text: 'Hello')
service = instance_double(TranslationService::DeepL, translate: translation, supported?: true)
service = instance_double(TranslationService::DeepL, translate: translation)
allow(TranslationService).to receive(:configured?).and_return(true)
allow(TranslationService).to receive(:configured).and_return(service)
Rails.cache.write('translation_service/languages', { 'es' => ['en'] })
post :create, params: { status_id: status.id }
end

@ -16,29 +16,6 @@ RSpec.describe TranslationService::DeepL do
)
end
describe '#supported?' do
it 'supports included languages as source and target languages' do
expect(service.supported?('uk', 'en')).to be true
end
it 'supports auto-detecting source language' do
expect(service.supported?(nil, 'en')).to be true
end
it 'supports "en" and "pt" as target languages though not included in language list' do
expect(service.supported?('uk', 'en')).to be true
expect(service.supported?('uk', 'pt')).to be true
end
it 'does not support non-included language as target language' do
expect(service.supported?('uk', 'nl')).to be false
end
it 'does not support non-included language as source language' do
expect(service.supported?('da', 'en')).to be false
end
end
describe '#translate' do
it 'returns translation with specified source language' do
stub_request(:post, 'https://api.deepl.com/v2/translate')
@ -63,13 +40,18 @@ RSpec.describe TranslationService::DeepL do
end
end
describe '#languages?' do
describe '#languages' do
it 'returns source languages' do
expect(service.send(:languages, 'source')).to eq ['en', 'uk', nil]
expect(service.languages.keys).to eq [nil, 'en', 'uk']
end
it 'returns target languages for each source language' do
expect(service.languages['en']).to eq %w(pt en-GB zh)
expect(service.languages['uk']).to eq %w(en pt en-GB zh)
end
it 'returns target languages' do
expect(service.send(:languages, 'target')).to eq %w(en-gb zh en pt)
it 'returns target languages for auto-detection' do
expect(service.languages[nil]).to eq %w(en pt en-GB zh)
end
end

@ -7,41 +7,24 @@ RSpec.describe TranslationService::LibreTranslate do
before do
stub_request(:get, 'https://libretranslate.example.com/languages').to_return(
body: '[{"code": "en","name": "English","targets": ["de","es"]},{"code": "da","name": "Danish","targets": ["en","de"]}]'
body: '[{"code": "en","name": "English","targets": ["de","en","es"]},{"code": "da","name": "Danish","targets": ["en","pt"]}]'
)
end
describe '#supported?' do
it 'supports included language pair' do
expect(service.supported?('en', 'de')).to be true
end
it 'does not support reversed language pair' do
expect(service.supported?('de', 'en')).to be false
end
it 'supports auto-detecting source language' do
expect(service.supported?(nil, 'de')).to be true
end
it 'does not support auto-detecting for unsupported target language' do
expect(service.supported?(nil, 'pt')).to be false
end
end
describe '#languages' do
subject(:languages) { service.send(:languages) }
subject(:languages) { service.languages }
it 'includes supported source languages' do
it 'returns source languages' do
expect(languages.keys).to eq ['en', 'da', nil]
end
it 'includes supported target languages for source language' do
it 'returns target languages for each source language' do
expect(languages['en']).to eq %w(de es)
expect(languages['da']).to eq %w(en pt)
end
it 'includes supported target languages for auto-detected language' do
expect(languages[nil]).to eq %w(de es en)
it 'returns target languages for auto-detected language' do
expect(languages[nil]).to eq %w(de en es pt)
end
end

@ -114,85 +114,6 @@ RSpec.describe Status, type: :model do
end
end
describe '#translatable?' do
before do
allow(TranslationService).to receive(:configured?).and_return(true)
allow(TranslationService).to receive(:configured).and_return(TranslationService.new)
allow(TranslationService.configured).to receive(:supported?).with('es', 'en').and_return(true)
subject.language = 'es'
subject.visibility = :public
end
context 'all conditions are satisfied' do
it 'returns true' do
expect(subject.translatable?).to be true
end
end
context 'translation service is not configured' do
it 'returns false' do
allow(TranslationService).to receive(:configured?).and_return(false)
allow(TranslationService).to receive(:configured).and_raise(TranslationService::NotConfiguredError)
expect(subject.translatable?).to be false
end
end
context 'status language is nil' do
it 'returns true' do
subject.language = nil
allow(TranslationService.configured).to receive(:supported?).with(nil, 'en').and_return(true)
expect(subject.translatable?).to be true
end
end
context 'status language is same as default locale' do
it 'returns false' do
subject.language = I18n.locale
expect(subject.translatable?).to be false
end
end
context 'status language is unsupported' do
it 'returns false' do
subject.language = 'af'
allow(TranslationService.configured).to receive(:supported?).with('af', 'en').and_return(false)
expect(subject.translatable?).to be false
end
end
context 'default locale is unsupported' do
it 'returns false' do
allow(TranslationService.configured).to receive(:supported?).with('es', 'af').and_return(false)
I18n.with_locale('af') do
expect(subject.translatable?).to be false
end
end
end
context 'default locale has region' do
it 'returns true' do
I18n.with_locale('en-GB') do
expect(subject.translatable?).to be true
end
end
end
context 'status text is blank' do
it 'returns false' do
subject.text = ' '
expect(subject.translatable?).to be false
end
end
context 'status visiblity is hidden' do
it 'returns false' do
subject.visibility = 'limited'
expect(subject.translatable?).to be false
end
end
end
describe '#content' do
it 'returns the text of the status if it is not a reblog' do
expect(subject.content).to eql subject.text

@ -3,7 +3,7 @@
require 'rails_helper'
describe InstancePresenter do
let(:instance_presenter) { InstancePresenter.new }
let(:instance_presenter) { described_class.new }
describe '#description' do
around do |example|

Loading…
Cancel
Save