From 6385ddb97f8d5add8d353bf221415e70611debdf Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 30 Jun 2019 16:10:43 +0200 Subject: [PATCH 01/15] Fix support for MP4 files that are actually M4V files (#11210) Resolve #11187 --- app/models/concerns/attachmentable.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/concerns/attachmentable.rb b/app/models/concerns/attachmentable.rb index 24f5968de4..7c78bb4569 100644 --- a/app/models/concerns/attachmentable.rb +++ b/app/models/concerns/attachmentable.rb @@ -60,7 +60,9 @@ module Attachmentable end def calculated_content_type(attachment) - Paperclip.run('file', '-b --mime :file', file: attachment.queued_for_write[:original].path).split(/[:;\s]+/).first.chomp + content_type = Paperclip.run('file', '-b --mime :file', file: attachment.queued_for_write[:original].path).split(/[:;\s]+/).first.chomp + content_type = 'video/mp4' if content_type == 'video/x-m4v' + content_type rescue Terrapin::CommandLineError '' end From a70732fd06f7eb3b746b471843061dbf542abf38 Mon Sep 17 00:00:00 2001 From: ThibG Date: Sun, 30 Jun 2019 16:11:21 +0200 Subject: [PATCH 02/15] =?UTF-8?q?Fix=20expiration=20date=20of=20filters=20?= =?UTF-8?q?being=20set=20to=20=E2=80=9CNever=E2=80=9D=20when=20editing=20t?= =?UTF-8?q?hem=20(#11204)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When editing a custom filter, select the shortest preset duration that still covers the remaining time of that filter. Fixes #9506 --- app/models/custom_filter.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/models/custom_filter.rb b/app/models/custom_filter.rb index 342207a55d..382562fb84 100644 --- a/app/models/custom_filter.rb +++ b/app/models/custom_filter.rb @@ -35,6 +35,13 @@ class CustomFilter < ApplicationRecord before_validation :clean_up_contexts after_commit :remove_cache + def expires_in + return @expires_in if defined?(@expires_in) + return nil if expires_at.nil? + + [30.minutes, 1.hour, 6.hours, 12.hours, 1.day, 1.week].find { |expires_in| expires_in.from_now >= expires_at } + end + private def clean_up_contexts From 05cead26f865a8d1283c7e10ba1669eddf3b0ccd Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2019 21:23:40 +0900 Subject: [PATCH 03/15] Bump webpack-cli from 3.3.4 to 3.3.5 (#11221) Bumps [webpack-cli](https://github.com/webpack/webpack-cli) from 3.3.4 to 3.3.5. - [Release notes](https://github.com/webpack/webpack-cli/releases) - [Changelog](https://github.com/webpack/webpack-cli/blob/master/CHANGELOG.md) - [Commits](https://github.com/webpack/webpack-cli/compare/v3.3.4...v3.3.5) Signed-off-by: dependabot-preview[bot] --- package.json | 2 +- yarn.lock | 179 ++++++++++++++++++++++++++++++++++----------------- 2 files changed, 122 insertions(+), 59 deletions(-) diff --git a/package.json b/package.json index b45fad6377..a87ec891de 100644 --- a/package.json +++ b/package.json @@ -161,7 +161,7 @@ "webpack": "^4.34.0", "webpack-assets-manifest": "^3.1.1", "webpack-bundle-analyzer": "^3.3.2", - "webpack-cli": "^3.3.4", + "webpack-cli": "^3.3.5", "webpack-merge": "^4.2.1", "websocket.js": "^0.1.12" }, diff --git a/yarn.lock b/yarn.lock index 2c402782c8..f48f484c6c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2270,6 +2270,15 @@ caseless@~0.12.0: resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw= +chalk@2.4.2, chalk@^2.0, chalk@^2.0.0, chalk@^2.0.1, chalk@^2.1.0, chalk@^2.3.2, chalk@^2.4.1, chalk@^2.4.2: + version "2.4.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" + integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + chalk@^1.0.0, chalk@^1.1.1, chalk@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" @@ -2281,15 +2290,6 @@ chalk@^1.0.0, chalk@^1.1.1, chalk@^1.1.3: strip-ansi "^3.0.0" supports-color "^2.0.0" -chalk@^2.0, chalk@^2.0.0, chalk@^2.0.1, chalk@^2.1.0, chalk@^2.3.2, chalk@^2.4.1, chalk@^2.4.2: - version "2.4.2" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" - integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== - dependencies: - ansi-styles "^3.2.1" - escape-string-regexp "^1.0.5" - supports-color "^5.3.0" - chardet@^0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" @@ -2404,6 +2404,15 @@ cliui@^4.0.0: strip-ansi "^4.0.0" wrap-ansi "^2.0.0" +cliui@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/cliui/-/cliui-5.0.0.tgz#deefcfdb2e800784aa34f46fa08e06851c7bbbc5" + integrity sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA== + dependencies: + string-width "^3.1.0" + strip-ansi "^5.2.0" + wrap-ansi "^5.1.0" + clone-deep@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-2.0.2.tgz#00db3a1e173656730d1188c3d6aced6d7ea97713" @@ -2737,7 +2746,7 @@ cross-env@^5.1.4: cross-spawn "^6.0.5" is-windows "^1.0.0" -cross-spawn@^6.0.0, cross-spawn@^6.0.5: +cross-spawn@6.0.5, cross-spawn@^6.0.0, cross-spawn@^6.0.5: version "6.0.5" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4" integrity sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ== @@ -3424,7 +3433,7 @@ end-of-stream@^1.0.0, end-of-stream@^1.1.0: dependencies: once "^1.4.0" -enhanced-resolve@^4.1.0: +enhanced-resolve@4.1.0, enhanced-resolve@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-4.1.0.tgz#41c7e0bfdfe74ac1ffe1e57ad6a5c6c9f3742a7f" integrity sha512-F/7vkyTtyc/llOIn8oWclcB25KdRaiPBpZYDgJHgh/UHtpgT2p2eldQgtQnLtUvfMKPKxbRaQM/hHkvLHt1Vng== @@ -4200,13 +4209,13 @@ find-up@^3.0.0: dependencies: locate-path "^3.0.0" -findup-sync@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-2.0.0.tgz#9326b1488c22d1a6088650a86901b2d9a90a2cbc" - integrity sha1-kyaxSIwi0aYIhlCoaQGy2akKLLw= +findup-sync@3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-3.0.0.tgz#17b108f9ee512dfb7a5c7f3c8b27ea9e1a9c08d1" + integrity sha512-YbffarhcicEhOrm4CtrwdKBdCuz576RLdhJDsIfvNtxUuhdRet1qZcsMjqbePtAseKdAnDyM/IyXbu7PRPRLYg== dependencies: detect-file "^1.0.0" - is-glob "^3.1.0" + is-glob "^4.0.0" micromatch "^3.0.4" resolve-dir "^1.0.1" @@ -4429,6 +4438,11 @@ get-caller-file@^1.0.1: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.3.tgz#f978fa4c90d1dfe7ff2d6beda2a515e713bdcf4a" integrity sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w== +get-caller-file@^2.0.1: + version "2.0.5" + resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" + integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== + get-stream@^4.0.0: version "4.1.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5" @@ -4468,6 +4482,13 @@ glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@~7.1.1: once "^1.3.0" path-is-absolute "^1.0.0" +global-modules@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-2.0.0.tgz#997605ad2345f27f51539bea26574421215c7780" + integrity sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A== + dependencies: + global-prefix "^3.0.0" + global-modules@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-1.0.0.tgz#6d770f0eb523ac78164d72b5e71a8877265cc3ea" @@ -4488,6 +4509,15 @@ global-prefix@^1.0.1: is-windows "^1.0.1" which "^1.2.14" +global-prefix@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/global-prefix/-/global-prefix-3.0.0.tgz#fc85f73064df69f50421f47f883fe5b913ba9b97" + integrity sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg== + dependencies: + ini "^1.3.5" + kind-of "^6.0.2" + which "^1.3.1" + globals@^11.1.0, globals@^11.7.0: version "11.12.0" resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" @@ -4913,7 +4943,7 @@ import-from@^2.1.0: dependencies: resolve-from "^3.0.0" -import-local@^2.0.0: +import-local@2.0.0, import-local@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/import-local/-/import-local-2.0.0.tgz#55070be38a5993cf18ef6db7e961f5bee5c5a09d" integrity sha512-b6s04m3O+s3CGSbqDIyP4R6aAwAeYlVq9+WUWep6iHa8ETRf9yei1U48C5MmfJmV9AiLYYBKPMq/W+/WRpQmCQ== @@ -4962,7 +4992,7 @@ inherits@2.0.1: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.1.tgz#b17d08d326b4423e568eff719f91b0b1cbdf69f1" integrity sha1-sX0I0ya0Qj5Wjv9xn5GwscvfafE= -ini@^1.3.4, ini@~1.3.0: +ini@^1.3.4, ini@^1.3.5, ini@~1.3.0: version "1.3.5" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927" integrity sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw== @@ -5013,7 +5043,7 @@ internal-ip@^4.3.0: default-gateway "^4.2.0" ipaddr.js "^1.9.0" -interpret@^1.1.0: +interpret@1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.2.0.tgz#d5061a6224be58e8083985f5014d844359576296" integrity sha512-mT34yGKMNceBQUoVn7iCDKDntA7SC6gycMAWzGx1z/CMCTV7b2AAtXlo3nRyHZ1FelRkQbQjprHSYGwzLtkVbw== @@ -6111,7 +6141,7 @@ loader-utils@0.2.x: json5 "^0.5.0" object-assign "^4.0.1" -loader-utils@^1.0.1, loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.2, loader-utils@^1.2.3: +loader-utils@1.2.3, loader-utils@^1.0.1, loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.2, loader-utils@^1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.2.3.tgz#1ff5dc6911c9f0a062531a4c04b609406108c2c7" integrity sha512-fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA== @@ -7026,7 +7056,7 @@ os-homedir@^1.0.0: resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" integrity sha1-/7xJiDNuDoM94MFox+8VISGqf7M= -os-locale@^3.0.0: +os-locale@^3.0.0, os-locale@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/os-locale/-/os-locale-3.1.0.tgz#a802a6ee17f24c10483ab9935719cef4ed16bf1a" integrity sha512-Z8l3R4wYWM40/52Z+S265okfFj8Kt2cC2MKY+xNi3kFs+XGI7WXu/I309QQQYbRW4ijiZ+yxs9pqEhJh0DqW3Q== @@ -7823,11 +7853,6 @@ prepend-http@^1.0.0: resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-1.0.4.tgz#d4f4562b0ce3696e41ac52d0e002e57a635dc6dc" integrity sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw= -prettier@^1.17.0: - version "1.18.2" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.18.2.tgz#6823e7c5900017b4bd3acf46fe9ac4b4d7bda9ea" - integrity sha512-OeHeMc0JhFE9idD4ZdtNibzY0+TPHSpSSb9h8FqtP+YnoZZ1sl8Vc9b1sasjfymH3SonAF4QcA2+mzHPhMvIiw== - pretty-format@^24.8.0: version "24.8.0" resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-24.8.0.tgz#8dae7044f58db7cb8be245383b565a963e3c27f2" @@ -8642,6 +8667,11 @@ require-main-filename@^1.0.1: resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-1.0.1.tgz#97f717b69d48784f5f526a6c5aa8ffdda055a4d1" integrity sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE= +require-main-filename@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b" + integrity sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg== + require-package-name@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/require-package-name/-/require-package-name-2.0.1.tgz#c11e97276b65b8e2923f75dabf5fb2ef0c3841b9" @@ -9403,7 +9433,7 @@ string-width@^1.0.1: is-fullwidth-code-point "^2.0.0" strip-ansi "^4.0.0" -string-width@^3.0.0: +string-width@^3.0.0, string-width@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/string-width/-/string-width-3.1.0.tgz#22767be21b62af1081574306f69ac51b62203961" integrity sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w== @@ -9456,7 +9486,7 @@ strip-ansi@^4.0.0: dependencies: ansi-regex "^3.0.0" -strip-ansi@^5.0.0, strip-ansi@^5.1.0: +strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-5.2.0.tgz#8c9a536feb6afc962bdfa5b104a5091c1ad9c0ae" integrity sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA== @@ -9507,6 +9537,13 @@ substring-trie@^1.0.2: resolved "https://registry.yarnpkg.com/substring-trie/-/substring-trie-1.0.2.tgz#7b42592391628b4f2cb17365c6cce4257c7b7af5" integrity sha1-e0JZI5Fii08ssXNlxszkJXx7evU= +supports-color@6.1.0, supports-color@^6.0.0, supports-color@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-6.1.0.tgz#0764abc69c63d5ac842dd4867e8d025e880df8f3" + integrity sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ== + dependencies: + has-flag "^3.0.0" + supports-color@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" @@ -9519,20 +9556,13 @@ supports-color@^3.2.3: dependencies: has-flag "^1.0.0" -supports-color@^5.3.0, supports-color@^5.5.0: +supports-color@^5.3.0: version "5.5.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== dependencies: has-flag "^3.0.0" -supports-color@^6.0.0, supports-color@^6.1.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-6.1.0.tgz#0764abc69c63d5ac842dd4867e8d025e880df8f3" - integrity sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ== - dependencies: - has-flag "^3.0.0" - svgo@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/svgo/-/svgo-1.1.1.tgz#12384b03335bcecd85cfa5f4e3375fed671cb985" @@ -10009,10 +10039,10 @@ uuid@^3.0.1, uuid@^3.1.0, uuid@^3.3.2: resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== -v8-compile-cache@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.0.2.tgz#a428b28bb26790734c4fc8bc9fa106fccebf6a6c" - integrity sha512-1wFuMUIM16MDJRCrpbpuEPTUGmM5QMUg0cr3KFwra2XgOgFcPGDQHDh3CszSCD2Zewc/dh/pamNEW8CbfDebUw== +v8-compile-cache@2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.0.3.tgz#00f7494d2ae2b688cfe2899df6ed2c54bef91dbe" + integrity sha512-CNmdbwQMBjwr9Gsmohvm0pbL954tJrNzf6gWL3K+QMQf00PF7ERGrEiLgjuU3mKreLC2MeGhUsNV9ybTbLgd3w== validate-npm-package-license@^3.0.1: version "3.0.4" @@ -10134,23 +10164,22 @@ webpack-bundle-analyzer@^3.3.2: opener "^1.5.1" ws "^6.0.0" -webpack-cli@^3.3.4: - version "3.3.4" - resolved "https://registry.yarnpkg.com/webpack-cli/-/webpack-cli-3.3.4.tgz#de27e281c48a897b8c219cb093e261d5f6afe44a" - integrity sha512-ubJGQEKMtBSpT+LiL5hXvn2GIOWiRWItR1DGUqJRhwRBeGhpRXjvF5f0erqdRJLErkfqS5/Ldkkedh4AL5Q1ZQ== - dependencies: - chalk "^2.4.1" - cross-spawn "^6.0.5" - enhanced-resolve "^4.1.0" - findup-sync "^2.0.0" - global-modules "^1.0.0" - import-local "^2.0.0" - interpret "^1.1.0" - loader-utils "^1.1.0" - prettier "^1.17.0" - supports-color "^5.5.0" - v8-compile-cache "^2.0.2" - yargs "^12.0.5" +webpack-cli@^3.3.5: + version "3.3.5" + resolved "https://registry.yarnpkg.com/webpack-cli/-/webpack-cli-3.3.5.tgz#f4d1238a66a2843d9cebf189835ea22142e72767" + integrity sha512-w0j/s42c5UhchwTmV/45MLQnTVwRoaUTu9fM5LuyOd/8lFoCNCELDogFoecx5NzRUndO0yD/gF2b02XKMnmAWQ== + dependencies: + chalk "2.4.2" + cross-spawn "6.0.5" + enhanced-resolve "4.1.0" + findup-sync "3.0.0" + global-modules "2.0.0" + import-local "2.0.0" + interpret "1.2.0" + loader-utils "1.2.3" + supports-color "6.1.0" + v8-compile-cache "2.0.3" + yargs "13.2.4" webpack-dev-middleware@^3.7.0: version "3.7.0" @@ -10312,7 +10341,7 @@ which-module@^2.0.0: resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" integrity sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho= -which@^1.2.14, which@^1.2.9, which@^1.3.0: +which@^1.2.14, which@^1.2.9, which@^1.3.0, which@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ== @@ -10351,6 +10380,15 @@ wrap-ansi@^2.0.0: string-width "^1.0.1" strip-ansi "^3.0.1" +wrap-ansi@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-5.1.0.tgz#1fd1f67235d5b6d0fee781056001bfb694c03b09" + integrity sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q== + dependencies: + ansi-styles "^3.2.0" + string-width "^3.0.0" + strip-ansi "^5.0.0" + wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" @@ -10421,6 +10459,14 @@ yargs-parser@^11.1.1: camelcase "^5.0.0" decamelize "^1.2.0" +yargs-parser@^13.1.0: + version "13.1.1" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-13.1.1.tgz#d26058532aa06d365fe091f6a1fc06b2f7e5eca0" + integrity sha512-oVAVsHz6uFrg3XQheFII8ESO2ssAf9luWuAd6Wexsu4F3OtIW0o8IribPXYrD4WC24LWtPrJlGy87y5udK+dxQ== + dependencies: + camelcase "^5.0.0" + decamelize "^1.2.0" + yargs@12.0.5, yargs@^12.0.2, yargs@^12.0.5: version "12.0.5" resolved "https://registry.yarnpkg.com/yargs/-/yargs-12.0.5.tgz#05f5997b609647b64f66b81e3b4b10a368e7ad13" @@ -10438,3 +10484,20 @@ yargs@12.0.5, yargs@^12.0.2, yargs@^12.0.5: which-module "^2.0.0" y18n "^3.2.1 || ^4.0.0" yargs-parser "^11.1.1" + +yargs@13.2.4: + version "13.2.4" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-13.2.4.tgz#0b562b794016eb9651b98bd37acf364aa5d6dc83" + integrity sha512-HG/DWAJa1PAnHT9JAhNa8AbAv3FPaiLzioSjCcmuXXhP8MlpHO5vwls4g4j6n30Z74GVQj8Xa62dWVx1QCGklg== + dependencies: + cliui "^5.0.0" + find-up "^3.0.0" + get-caller-file "^2.0.1" + os-locale "^3.1.0" + require-directory "^2.1.1" + require-main-filename "^2.0.0" + set-blocking "^2.0.0" + string-width "^3.0.0" + which-module "^2.0.0" + y18n "^4.0.0" + yargs-parser "^13.1.0" From f556f363dbf84e7e7e250fcf803d9889b46ff461 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2019 21:29:56 +0900 Subject: [PATCH 04/15] Bump eslint-plugin-promise from 4.1.1 to 4.2.1 (#11223) Bumps [eslint-plugin-promise](https://github.com/xjamundx/eslint-plugin-promise) from 4.1.1 to 4.2.1. - [Release notes](https://github.com/xjamundx/eslint-plugin-promise/releases) - [Changelog](https://github.com/xjamundx/eslint-plugin-promise/blob/master/CHANGELOG.md) - [Commits](https://github.com/xjamundx/eslint-plugin-promise/commits) Signed-off-by: dependabot-preview[bot] --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index a87ec891de..358f5b1563 100644 --- a/package.json +++ b/package.json @@ -173,7 +173,7 @@ "eslint": "^5.16.0", "eslint-plugin-import": "~2.17.3", "eslint-plugin-jsx-a11y": "~6.2.1", - "eslint-plugin-promise": "~4.1.1", + "eslint-plugin-promise": "~4.2.1", "eslint-plugin-react": "~7.12.1", "jest": "^24.8.0", "raf": "^3.4.1", diff --git a/yarn.lock b/yarn.lock index f48f484c6c..6698f6a19e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3693,10 +3693,10 @@ eslint-plugin-jsx-a11y@~6.2.1: has "^1.0.3" jsx-ast-utils "^2.0.1" -eslint-plugin-promise@~4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/eslint-plugin-promise/-/eslint-plugin-promise-4.1.1.tgz#1e08cb68b5b2cd8839f8d5864c796f56d82746db" - integrity sha512-faAHw7uzlNPy7b45J1guyjazw28M+7gJokKUjC5JSFoYfUEyy6Gw/i7YQvmv2Yk00sUjWcmzXQLpU1Ki/C2IZQ== +eslint-plugin-promise@~4.2.1: + version "4.2.1" + resolved "https://registry.yarnpkg.com/eslint-plugin-promise/-/eslint-plugin-promise-4.2.1.tgz#845fd8b2260ad8f82564c1222fce44ad71d9418a" + integrity sha512-VoM09vT7bfA7D+upt+FjeBO5eHIJQBUWki1aPvB+vbNiHS3+oGIJGIeyBtKQTME6UPXXy3vV07OL1tHd3ANuDw== eslint-plugin-react@~7.12.1: version "7.12.1" From 6dcda24de27b1fc956f057317ef918d72c1c64ca Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2019 22:33:54 +0900 Subject: [PATCH 05/15] Bump bullet from 6.0.0 to 6.0.1 (#11228) Bumps [bullet](https://github.com/flyerhzm/bullet) from 6.0.0 to 6.0.1. - [Release notes](https://github.com/flyerhzm/bullet/releases) - [Changelog](https://github.com/flyerhzm/bullet/blob/master/CHANGELOG.md) - [Commits](https://github.com/flyerhzm/bullet/compare/6.0.0...6.0.1) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 8b0c93b1ea..5034c14119 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -106,7 +106,7 @@ GEM brakeman (4.5.1) browser (2.5.3) builder (3.2.3) - bullet (6.0.0) + bullet (6.0.1) activesupport (>= 3.0.0) uniform_notifier (~> 1.11) bundler-audit (0.6.1) From aa1d8a4aad90109df20b99a625818644f5d11a83 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2019 22:34:25 +0900 Subject: [PATCH 06/15] Bump memory_profiler from 0.9.13 to 0.9.14 (#11227) Bumps [memory_profiler](https://github.com/SamSaffron/memory_profiler) from 0.9.13 to 0.9.14. - [Release notes](https://github.com/SamSaffron/memory_profiler/releases) - [Changelog](https://github.com/SamSaffron/memory_profiler/blob/master/CHANGELOG.md) - [Commits](https://github.com/SamSaffron/memory_profiler/compare/v0.9.13...v0.9.14) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 5034c14119..8854b5fce3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -336,7 +336,7 @@ GEM mimemagic (~> 0.3.2) mario-redis-lock (1.2.1) redis (>= 3.0.5) - memory_profiler (0.9.13) + memory_profiler (0.9.14) method_source (0.9.2) microformats (4.1.0) json (~> 2.1) From 0b527c27c7ca4a4739b6e0234fa5735c0a553f74 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2019 22:34:51 +0900 Subject: [PATCH 07/15] Bump capybara from 3.24.0 to 3.25.0 (#11225) Bumps [capybara](https://github.com/teamcapybara/capybara) from 3.24.0 to 3.25.0. - [Release notes](https://github.com/teamcapybara/capybara/releases) - [Changelog](https://github.com/teamcapybara/capybara/blob/master/History.md) - [Commits](https://github.com/teamcapybara/capybara/compare/3.24.0...3.25.0) Signed-off-by: dependabot-preview[bot] --- Gemfile | 2 +- Gemfile.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index 4d63cca37f..42499cd2c2 100644 --- a/Gemfile +++ b/Gemfile @@ -108,7 +108,7 @@ group :production, :test do end group :test do - gem 'capybara', '~> 3.24' + gem 'capybara', '~> 3.25' gem 'climate_control', '~> 0.2' gem 'faker', '~> 1.9' gem 'microformats', '~> 4.1' diff --git a/Gemfile.lock b/Gemfile.lock index 8854b5fce3..4a4e86e728 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -129,7 +129,7 @@ GEM sshkit (~> 1.3) capistrano-yarn (2.0.2) capistrano (~> 3.0) - capybara (3.24.0) + capybara (3.25.0) addressable mini_mime (>= 0.1.3) nokogiri (~> 1.8) @@ -420,7 +420,7 @@ GEM pry (~> 0.10) pry-rails (0.3.9) pry (>= 0.10.4) - public_suffix (3.1.0) + public_suffix (3.1.1) puma (3.12.1) pundit (2.0.1) activesupport (>= 3.0.0) @@ -660,7 +660,7 @@ DEPENDENCIES capistrano-rails (~> 1.4) capistrano-rbenv (~> 2.1) capistrano-yarn (~> 2.0) - capybara (~> 3.24) + capybara (~> 3.25) charlock_holmes (~> 0.7.6) chewy (~> 5.0) cld3 (~> 3.2.4) From eda8ac5298eb557844b36ec9100690c92b7ac297 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2019 22:36:45 +0900 Subject: [PATCH 08/15] Bump rubocop from 0.71.0 to 0.72.0 (#11229) Bumps [rubocop](https://github.com/rubocop-hq/rubocop) from 0.71.0 to 0.72.0. - [Release notes](https://github.com/rubocop-hq/rubocop/releases) - [Changelog](https://github.com/rubocop-hq/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop-hq/rubocop/compare/v0.71.0...v0.72.0) Signed-off-by: dependabot-preview[bot] --- Gemfile | 2 +- Gemfile.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index 42499cd2c2..e99a808d40 100644 --- a/Gemfile +++ b/Gemfile @@ -128,7 +128,7 @@ group :development do gem 'letter_opener', '~> 1.7' gem 'letter_opener_web', '~> 1.3' gem 'memory_profiler' - gem 'rubocop', '~> 0.71', require: false + gem 'rubocop', '~> 0.72', require: false gem 'rubocop-rails', '~> 2.0', require: false gem 'brakeman', '~> 4.5', require: false gem 'bundler-audit', '~> 0.6', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 4a4e86e728..c8856ccc56 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -287,7 +287,7 @@ GEM idn-ruby (0.1.0) ipaddress (0.8.3) iso-639 (0.2.8) - jaro_winkler (1.5.2) + jaro_winkler (1.5.3) jmespath (1.4.0) json (2.1.0) json-ld (3.0.2) @@ -524,7 +524,7 @@ GEM rspec-core (~> 3.0, >= 3.0.0) sidekiq (>= 2.4.0) rspec-support (3.8.0) - rubocop (0.71.0) + rubocop (0.72.0) jaro_winkler (~> 1.5.1) parallel (~> 1.10) parser (>= 2.6) @@ -737,7 +737,7 @@ DEPENDENCIES rqrcode (~> 0.10) rspec-rails (~> 3.8) rspec-sidekiq (~> 3.0) - rubocop (~> 0.71) + rubocop (~> 0.72) rubocop-rails (~> 2.0) sanitize (~> 5.0) sidekiq (~> 5.2) From 9ac82421f81b9eacb38cc49b56a4b25de157b0e6 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 1 Jul 2019 19:13:30 +0200 Subject: [PATCH 09/15] Change domain block behaviour to prevent creation of accounts from suspended domains (#11219) --- app/services/activitypub/process_account_service.rb | 4 +++- app/services/resolve_account_service.rb | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 05c017bdf5..3857e7c16d 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -15,6 +15,8 @@ class ActivityPub::ProcessAccountService < BaseService @domain = domain @collections = {} + return if auto_suspend? + RedisLock.acquire(lock_options) do |lock| if lock.acquired? @account = Account.find_remote(@username, @domain) @@ -55,7 +57,7 @@ class ActivityPub::ProcessAccountService < BaseService @account.domain = @domain @account.private_key = nil @account.suspended_at = domain_block.created_at if auto_suspend? - @account.silenced_at = domain_block.created_at if auto_silence? + @account.silenced_at = domain_block.created_at if auto_silence? end def update_account diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index 57c9ccfe1b..e557706da5 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -48,7 +48,7 @@ class ResolveAccountService < BaseService return end - return if links_missing? + return if links_missing? || auto_suspend? return Account.find_local(@username) if TagManager.instance.local_domain?(@domain) RedisLock.acquire(lock_options) do |lock| From e782fcef4f78bba36bd06d74f38ac849de91d50d Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 2 Jul 2019 00:34:38 +0200 Subject: [PATCH 10/15] Add request pool to improve delivery performance (#10353) * Add request pool to improve delivery performance Fix #7909 * Ensure connection is closed when exception interrupts execution * Remove Timeout#timeout from socket connection * Fix infinite retrial loop on HTTP::ConnectionError * Close sockets on failure, reduce idle time to 90 seconds * Add MAX_REQUEST_POOL_SIZE option to limit concurrent connections to the same server * Use a shared pool size, 512 by default, to stay below open file limit * Add some tests * Add more tests * Reduce MAX_IDLE_TIME from 90 to 30 seconds, reap every 30 seconds * Use a shared pool that returns preferred connection but re-purposes other ones when needed * Fix wrong connection being returned on subsequent calls within the same thread * Reduce mutex calls on flushes from 2 to 1 and add test for reaping --- Gemfile | 1 + Gemfile.lock | 1 + .../connection_pool/shared_connection_pool.rb | 63 ++++++++++ app/lib/connection_pool/shared_timed_stack.rb | 95 +++++++++++++++ app/lib/request.rb | 70 ++++++++--- app/lib/request_pool.rb | 114 ++++++++++++++++++ app/workers/activitypub/delivery_worker.rb | 15 ++- .../shared_connection_pool_spec.rb | 28 +++++ .../shared_timed_stack_spec.rb | 61 ++++++++++ spec/lib/request_pool_spec.rb | 63 ++++++++++ 10 files changed, 488 insertions(+), 23 deletions(-) create mode 100644 app/lib/connection_pool/shared_connection_pool.rb create mode 100644 app/lib/connection_pool/shared_timed_stack.rb create mode 100644 app/lib/request_pool.rb create mode 100644 spec/lib/connection_pool/shared_connection_pool_spec.rb create mode 100644 spec/lib/connection_pool/shared_timed_stack_spec.rb create mode 100644 spec/lib/request_pool_spec.rb diff --git a/Gemfile b/Gemfile index e99a808d40..2fcb603c33 100644 --- a/Gemfile +++ b/Gemfile @@ -148,3 +148,4 @@ group :production do end gem 'concurrent-ruby', require: false +gem 'connection_pool', require: false diff --git a/Gemfile.lock b/Gemfile.lock index c8856ccc56..6043a2fa9e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -666,6 +666,7 @@ DEPENDENCIES cld3 (~> 3.2.4) climate_control (~> 0.2) concurrent-ruby + connection_pool derailed_benchmarks devise (~> 4.6) devise-two-factor (~> 3.0) diff --git a/app/lib/connection_pool/shared_connection_pool.rb b/app/lib/connection_pool/shared_connection_pool.rb new file mode 100644 index 0000000000..2865a4108d --- /dev/null +++ b/app/lib/connection_pool/shared_connection_pool.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'connection_pool' +require_relative './shared_timed_stack' + +class ConnectionPool::SharedConnectionPool < ConnectionPool + def initialize(options = {}, &block) + super(options, &block) + + @available = ConnectionPool::SharedTimedStack.new(@size, &block) + end + + delegate :size, :flush, to: :@available + + def with(preferred_tag, options = {}) + Thread.handle_interrupt(Exception => :never) do + conn = checkout(preferred_tag, options) + + begin + Thread.handle_interrupt(Exception => :immediate) do + yield conn + end + ensure + checkin(preferred_tag) + end + end + end + + def checkout(preferred_tag, options = {}) + if ::Thread.current[key(preferred_tag)] + ::Thread.current[key_count(preferred_tag)] += 1 + ::Thread.current[key(preferred_tag)] + else + ::Thread.current[key_count(preferred_tag)] = 1 + ::Thread.current[key(preferred_tag)] = @available.pop(preferred_tag, options[:timeout] || @timeout) + end + end + + def checkin(preferred_tag) + if ::Thread.current[key(preferred_tag)] + if ::Thread.current[key_count(preferred_tag)] == 1 + @available.push(::Thread.current[key(preferred_tag)]) + ::Thread.current[key(preferred_tag)] = nil + else + ::Thread.current[key_count(preferred_tag)] -= 1 + end + else + raise ConnectionPool::Error, 'no connections are checked out' + end + + nil + end + + private + + def key(tag) + :"#{@key}-#{tag}" + end + + def key_count(tag) + :"#{@key_count}-#{tag}" + end +end diff --git a/app/lib/connection_pool/shared_timed_stack.rb b/app/lib/connection_pool/shared_timed_stack.rb new file mode 100644 index 0000000000..14a5285c45 --- /dev/null +++ b/app/lib/connection_pool/shared_timed_stack.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +class ConnectionPool::SharedTimedStack + def initialize(max = 0, &block) + @create_block = block + @max = max + @created = 0 + @queue = [] + @tagged_queue = Hash.new { |hash, key| hash[key] = [] } + @mutex = Mutex.new + @resource = ConditionVariable.new + end + + def push(connection) + @mutex.synchronize do + store_connection(connection) + @resource.broadcast + end + end + + alias << push + + def pop(preferred_tag, timeout = 5.0) + deadline = current_time + timeout + + @mutex.synchronize do + loop do + return fetch_preferred_connection(preferred_tag) unless @tagged_queue[preferred_tag].empty? + + connection = try_create(preferred_tag) + return connection if connection + + to_wait = deadline - current_time + raise Timeout::Error, "Waited #{timeout} sec" if to_wait <= 0 + + @resource.wait(@mutex, to_wait) + end + end + end + + def empty? + size.zero? + end + + def size + @mutex.synchronize do + @queue.size + end + end + + def flush + @mutex.synchronize do + @queue.delete_if do |connection| + delete = !connection.in_use && (connection.dead || connection.seconds_idle >= RequestPool::MAX_IDLE_TIME) + + if delete + @tagged_queue[connection.site].delete(connection) + connection.close + @created -= 1 + end + + delete + end + end + end + + private + + def try_create(preferred_tag) + if @created == @max && !@queue.empty? + throw_away_connection = @queue.pop + @tagged_queue[throw_away_connection.site].delete(throw_away_connection) + @create_block.call(preferred_tag) + elsif @created != @max + connection = @create_block.call(preferred_tag) + @created += 1 + connection + end + end + + def fetch_preferred_connection(preferred_tag) + connection = @tagged_queue[preferred_tag].pop + @queue.delete(connection) + connection + end + + def current_time + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + + def store_connection(connection) + @tagged_queue[connection.site].push(connection) + @queue.push(connection) + end +end diff --git a/app/lib/request.rb b/app/lib/request.rb index e555ae6a10..af49d6c77b 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -17,15 +17,21 @@ end class Request REQUEST_TARGET = '(request-target)' + # We enforce a 5s timeout on DNS resolving, 5s timeout on socket opening + # and 5s timeout on the TLS handshake, meaning the worst case should take + # about 15s in total + TIMEOUT = { connect: 5, read: 10, write: 10 }.freeze + include RoutingHelper def initialize(verb, url, **options) raise ArgumentError if url.blank? - @verb = verb - @url = Addressable::URI.parse(url).normalize - @options = options.merge(use_proxy? ? Rails.configuration.x.http_client_proxy : { socket_class: Socket }) - @headers = {} + @verb = verb + @url = Addressable::URI.parse(url).normalize + @http_client = options.delete(:http_client) + @options = options.merge(use_proxy? ? Rails.configuration.x.http_client_proxy : { socket_class: Socket }) + @headers = {} raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if block_hidden_service? @@ -50,15 +56,24 @@ class Request def perform begin - response = http_client.headers(headers).public_send(@verb, @url.to_s, @options) + response = http_client.public_send(@verb, @url.to_s, @options.merge(headers: headers)) rescue => e raise e.class, "#{e.message} on #{@url}", e.backtrace[0] end begin - yield response.extend(ClientLimit) if block_given? + response = response.extend(ClientLimit) + + # If we are using a persistent connection, we have to + # read every response to be able to move forward at all. + # However, simply calling #to_s or #flush may not be safe, + # as the response body, if malicious, could be too big + # for our memory. So we use the #body_with_limit method + response.body_with_limit if http_client.persistent? + + yield response if block_given? ensure - http_client.close + http_client.close unless http_client.persistent? end end @@ -76,6 +91,10 @@ class Request %w(http https).include?(parsed_url.scheme) && parsed_url.host.present? end + + def http_client + HTTP.use(:auto_inflate).timeout(:per_operation, TIMEOUT.dup).follow(max_hops: 2) + end end private @@ -116,16 +135,8 @@ class Request end end - def timeout - # We enforce a 1s timeout on DNS resolving, 10s timeout on socket opening - # and 5s timeout on the TLS handshake, meaning the worst case should take - # about 16s in total - - { connect: 5, read: 10, write: 10 } - end - def http_client - @http_client ||= HTTP.use(:auto_inflate).timeout(:per_operation, timeout).follow(max_hops: 2) + @http_client ||= Request.http_client end def use_proxy? @@ -169,20 +180,41 @@ class Request return super(host, *args) if thru_hidden_service?(host) outer_e = nil + port = args.first Resolv::DNS.open do |dns| dns.timeouts = 5 addresses = dns.getaddresses(host).take(2) - time_slot = 10.0 / addresses.size addresses.each do |address| begin raise Mastodon::HostValidationError if PrivateAddressCheck.private_address?(IPAddr.new(address.to_s)) - ::Timeout.timeout(time_slot, HTTP::TimeoutError) do - return super(address.to_s, *args) + sock = ::Socket.new(::Socket::AF_INET, ::Socket::SOCK_STREAM, 0) + sockaddr = ::Socket.pack_sockaddr_in(port, address.to_s) + + sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, 1) + + begin + sock.connect_nonblock(sockaddr) + rescue IO::WaitWritable + if IO.select(nil, [sock], nil, Request::TIMEOUT[:connect]) + begin + sock.connect_nonblock(sockaddr) + rescue Errno::EISCONN + # Yippee! + rescue + sock.close + raise + end + else + sock.close + raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds" + end end + + return sock rescue => e outer_e = e end diff --git a/app/lib/request_pool.rb b/app/lib/request_pool.rb new file mode 100644 index 0000000000..e5899a79aa --- /dev/null +++ b/app/lib/request_pool.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require_relative './connection_pool/shared_connection_pool' + +class RequestPool + def self.current + @current ||= RequestPool.new + end + + class Reaper + attr_reader :pool, :frequency + + def initialize(pool, frequency) + @pool = pool + @frequency = frequency + end + + def run + return unless frequency&.positive? + + Thread.new(frequency, pool) do |t, p| + loop do + sleep t + p.flush + end + end + end + end + + MAX_IDLE_TIME = 30 + WAIT_TIMEOUT = 5 + MAX_POOL_SIZE = ENV.fetch('MAX_REQUEST_POOL_SIZE', 512).to_i + + class Connection + attr_reader :site, :last_used_at, :created_at, :in_use, :dead, :fresh + + def initialize(site) + @site = site + @http_client = http_client + @last_used_at = nil + @created_at = current_time + @dead = false + @fresh = true + end + + def use + @last_used_at = current_time + @in_use = true + + retries = 0 + + begin + yield @http_client + rescue HTTP::ConnectionError + # It's possible the connection was closed, so let's + # try re-opening it once + + close + + if @fresh || retries.positive? + raise + else + @http_client = http_client + retries += 1 + retry + end + rescue StandardError + # If this connection raises errors of any kind, it's + # better if it gets reaped as soon as possible + + close + @dead = true + raise + end + ensure + @fresh = false + @in_use = false + end + + def seconds_idle + current_time - (@last_used_at || @created_at) + end + + def close + @http_client.close + end + + private + + def http_client + Request.http_client.persistent(@site, timeout: MAX_IDLE_TIME) + end + + def current_time + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + end + + def initialize + @pool = ConnectionPool::SharedConnectionPool.new(size: MAX_POOL_SIZE, timeout: WAIT_TIMEOUT) { |site| Connection.new(site) } + @reaper = Reaper.new(self, 30) + @reaper.run + end + + def with(site, &block) + @pool.with(site) do |connection| + ActiveSupport::Notifications.instrument('with.request_pool', miss: connection.fresh, host: connection.site) do + connection.use(&block) + end + end + end + + delegate :size, :flush, to: :@pool +end diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb index 5e4c391f0d..79f1e81536 100644 --- a/app/workers/activitypub/delivery_worker.rb +++ b/app/workers/activitypub/delivery_worker.rb @@ -17,6 +17,7 @@ class ActivityPub::DeliveryWorker @json = json @source_account = Account.find(source_account_id) @inbox_url = inbox_url + @host = Addressable::URI.parse(inbox_url).normalized_site perform_request @@ -28,16 +29,18 @@ class ActivityPub::DeliveryWorker private - def build_request - request = Request.new(:post, @inbox_url, body: @json) + def build_request(http_client) + request = Request.new(:post, @inbox_url, body: @json, http_client: http_client) request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with]) request.add_headers(HEADERS) end def perform_request light = Stoplight(@inbox_url) do - build_request.perform do |response| - raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) + request_pool.with(@host) do |http_client| + build_request(http_client).perform do |response| + raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) + end end end @@ -57,4 +60,8 @@ class ActivityPub::DeliveryWorker def failure_tracker @failure_tracker ||= DeliveryFailureTracker.new(@inbox_url) end + + def request_pool + RequestPool.current + end end diff --git a/spec/lib/connection_pool/shared_connection_pool_spec.rb b/spec/lib/connection_pool/shared_connection_pool_spec.rb new file mode 100644 index 0000000000..1144645580 --- /dev/null +++ b/spec/lib/connection_pool/shared_connection_pool_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe ConnectionPool::SharedConnectionPool do + class MiniConnection + attr_reader :site + + def initialize(site) + @site = site + end + end + + subject { described_class.new(size: 5, timeout: 5) { |site| MiniConnection.new(site) } } + + describe '#with' do + it 'runs a block with a connection' do + block_run = false + + subject.with('foo') do |connection| + expect(connection).to be_a MiniConnection + block_run = true + end + + expect(block_run).to be true + end + end +end diff --git a/spec/lib/connection_pool/shared_timed_stack_spec.rb b/spec/lib/connection_pool/shared_timed_stack_spec.rb new file mode 100644 index 0000000000..f680c59667 --- /dev/null +++ b/spec/lib/connection_pool/shared_timed_stack_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe ConnectionPool::SharedTimedStack do + class MiniConnection + attr_reader :site + + def initialize(site) + @site = site + end + end + + subject { described_class.new(5) { |site| MiniConnection.new(site) } } + + describe '#push' do + it 'keeps the connection in the stack' do + subject.push(MiniConnection.new('foo')) + expect(subject.size).to eq 1 + end + end + + describe '#pop' do + it 'returns a connection' do + expect(subject.pop('foo')).to be_a MiniConnection + end + + it 'returns the same connection that was pushed in' do + connection = MiniConnection.new('foo') + subject.push(connection) + expect(subject.pop('foo')).to be connection + end + + it 'does not create more than maximum amount of connections' do + expect { 6.times { subject.pop('foo', 0) } }.to raise_error Timeout::Error + end + + it 'repurposes a connection for a different site when maximum amount is reached' do + 5.times { subject.push(MiniConnection.new('foo')) } + expect(subject.pop('bar')).to be_a MiniConnection + end + end + + describe '#empty?' do + it 'returns true when no connections on the stack' do + expect(subject.empty?).to be true + end + + it 'returns false when there are connections on the stack' do + subject.push(MiniConnection.new('foo')) + expect(subject.empty?).to be false + end + end + + describe '#size' do + it 'returns the number of connections on the stack' do + 2.times { subject.push(MiniConnection.new('foo')) } + expect(subject.size).to eq 2 + end + end +end diff --git a/spec/lib/request_pool_spec.rb b/spec/lib/request_pool_spec.rb new file mode 100644 index 0000000000..4a144d7c7f --- /dev/null +++ b/spec/lib/request_pool_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe RequestPool do + subject { described_class.new } + + describe '#with' do + it 'returns a HTTP client for a host' do + subject.with('http://example.com') do |http_client| + expect(http_client).to be_a HTTP::Client + end + end + + it 'returns the same instance of HTTP client within the same thread for the same host' do + test_client = nil + + subject.with('http://example.com') { |http_client| test_client = http_client } + expect(test_client).to_not be_nil + subject.with('http://example.com') { |http_client| expect(http_client).to be test_client } + end + + it 'returns different HTTP clients for different hosts' do + test_client = nil + + subject.with('http://example.com') { |http_client| test_client = http_client } + expect(test_client).to_not be_nil + subject.with('http://example.org') { |http_client| expect(http_client).to_not be test_client } + end + + it 'grows to the number of threads accessing it' do + stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'Hello!') + + subject + + threads = 20.times.map do |i| + Thread.new do + 20.times do + subject.with('http://example.com') do |http_client| + http_client.get('/').flush + end + end + end + end + + threads.map(&:join) + + expect(subject.size).to be > 1 + end + + it 'closes idle connections' do + stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'Hello!') + + subject.with('http://example.com') do |http_client| + http_client.get('/').flush + end + + expect(subject.size).to eq 1 + sleep RequestPool::MAX_IDLE_TIME + 30 + 1 + expect(subject.size).to eq 0 + end + end +end From a55fd408023c540993da8eba5b9c66144e44f31c Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 2 Jul 2019 00:36:16 +0200 Subject: [PATCH 11/15] When deleting & redrafting a poll, fill in closest expires_in (#11203) Use the smallest preset expires_in such that the new poll would not expire before the old one. In the typical case of a quick delete & redraft, this results in using the same poll duration. Fixes #10567 --- app/javascript/mastodon/reducers/compose.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/javascript/mastodon/reducers/compose.js b/app/javascript/mastodon/reducers/compose.js index 8cdd29bfeb..fae7522b26 100644 --- a/app/javascript/mastodon/reducers/compose.js +++ b/app/javascript/mastodon/reducers/compose.js @@ -195,6 +195,12 @@ const expandMentions = status => { return fragment.innerHTML; }; +const expiresInFromExpiresAt = expires_at => { + if (!expires_at) return 24 * 3600; + const delta = (new Date(expires_at).getTime() - Date.now()) / 1000; + return [300, 1800, 3600, 21600, 86400, 259200, 604800].find(expires_in => expires_in >= delta) || 24 * 3600; +}; + export default function compose(state = initialState, action) { switch(action.type) { case STORE_HYDRATE: @@ -353,7 +359,7 @@ export default function compose(state = initialState, action) { map.set('poll', ImmutableMap({ options: action.status.getIn(['poll', 'options']).map(x => x.get('title')), multiple: action.status.getIn(['poll', 'multiple']), - expires_in: 24 * 3600, + expires_in: expiresInFromExpiresAt(action.status.getIn(['poll', 'expires_at'])), })); } }); From 227d9297ba8ec401295df9e6194c68908e77cd8b Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 2 Jul 2019 00:59:53 +0200 Subject: [PATCH 12/15] Change ActivityPub::DeliveryWorker to not retry HTTP 501 errors (#11233) --- app/workers/activitypub/delivery_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb index 79f1e81536..818fd8f5d9 100644 --- a/app/workers/activitypub/delivery_worker.rb +++ b/app/workers/activitypub/delivery_worker.rb @@ -54,7 +54,7 @@ class ActivityPub::DeliveryWorker end def response_error_unsalvageable?(response) - (400...500).cover?(response.code) && ![401, 408, 429].include?(response.code) + response.code == 501 || ((400...500).cover?(response.code) && ![401, 408, 429].include?(response.code)) end def failure_tracker From 7391d223ab6e8128f8870b0a5af08d744070e45d Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 2 Jul 2019 01:01:17 +0200 Subject: [PATCH 13/15] Fix statsd UDP sockets not being cleaned up in Sidekiq (#11230) --- app/lib/sidekiq_error_handler.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/lib/sidekiq_error_handler.rb b/app/lib/sidekiq_error_handler.rb index 23785cf055..8eb6b942db 100644 --- a/app/lib/sidekiq_error_handler.rb +++ b/app/lib/sidekiq_error_handler.rb @@ -3,9 +3,11 @@ class SidekiqErrorHandler def call(*) yield - rescue Mastodon::HostValidationError => e - Rails.logger.error "#{e.class}: #{e.message}" - Rails.logger.error e.backtrace.join("\n") + rescue Mastodon::HostValidationError # Do not retry + ensure + socket = Thread.current[:statsd_socket] + socket&.close + Thread.current[:statsd_socket] = nil end end From 3ce4cdd552c94d8dad01bf6c7db778138305944e Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 2 Jul 2019 11:34:39 +0200 Subject: [PATCH 14/15] Remove unused StatsD code and expose StatsD as a global variable (#11232) The instrumentation code was used for StatsD metrics collection prior to the switch to the nsa gem and should have been removed at that point as it no longer does anything at all --- config/initializers/instrumentation.rb | 18 ------------------ config/initializers/statsd.rb | 6 +++--- 2 files changed, 3 insertions(+), 21 deletions(-) delete mode 100644 config/initializers/instrumentation.rb diff --git a/config/initializers/instrumentation.rb b/config/initializers/instrumentation.rb deleted file mode 100644 index 8483f2be2e..0000000000 --- a/config/initializers/instrumentation.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -instrumentation_hostname = ENV.fetch('INSTRUMENTATION_HOSTNAME') { 'localhost' } - -ActiveSupport::Notifications.subscribe(/process_action.action_controller/) do |*args| - event = ActiveSupport::Notifications::Event.new(*args) - controller = event.payload[:controller] - action = event.payload[:action] - format = event.payload[:format] || 'all' - format = 'all' if format == '*/*' - status = event.payload[:status] - key = "#{controller}.#{action}.#{format}.#{instrumentation_hostname}" - - ActiveSupport::Notifications.instrument :performance, action: :measure, measurement: "#{key}.total_duration", value: event.duration - ActiveSupport::Notifications.instrument :performance, action: :measure, measurement: "#{key}.db_time", value: event.payload[:db_runtime] - ActiveSupport::Notifications.instrument :performance, action: :measure, measurement: "#{key}.view_time", value: event.payload[:view_runtime] - ActiveSupport::Notifications.instrument :performance, measurement: "#{key}.status.#{status}" -end diff --git a/config/initializers/statsd.rb b/config/initializers/statsd.rb index ce83fd9de2..93ea1d1e4a 100644 --- a/config/initializers/statsd.rb +++ b/config/initializers/statsd.rb @@ -3,10 +3,10 @@ if ENV['STATSD_ADDR'].present? host, port = ENV['STATSD_ADDR'].split(':') - statsd = ::Statsd.new(host, port) - statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') } + $statsd = ::Statsd.new(host, port) + $statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') } - ::NSA.inform_statsd(statsd) do |informant| + ::NSA.inform_statsd($statsd) do |informant| informant.collect(:action_controller, :web) informant.collect(:active_record, :db) informant.collect(:active_support_cache, :cache) From eda4094171f51239c79bfcd68e36339ed0a3c533 Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 2 Jul 2019 16:03:54 +0200 Subject: [PATCH 15/15] Memoize ancestorIds and descendantIds in detailed status view (#11234) --- .../mastodon/features/status/index.js | 68 ++++++++++++------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/app/javascript/mastodon/features/status/index.js b/app/javascript/mastodon/features/status/index.js index 981eb9d58b..0422111ae3 100644 --- a/app/javascript/mastodon/features/status/index.js +++ b/app/javascript/mastodon/features/status/index.js @@ -4,6 +4,7 @@ import { connect } from 'react-redux'; import PropTypes from 'prop-types'; import classNames from 'classnames'; import ImmutablePropTypes from 'react-immutable-proptypes'; +import { createSelector } from 'reselect'; import { fetchStatus } from '../../actions/statuses'; import MissingIndicator from '../../components/missing_indicator'; import DetailedStatus from './components/detailed_status'; @@ -63,39 +64,58 @@ const messages = defineMessages({ const makeMapStateToProps = () => { const getStatus = makeGetStatus(); - const mapStateToProps = (state, props) => { - const status = getStatus(state, { id: props.params.statusId }); + const getAncestorsIds = createSelector([ + (_, { id }) => id, + state => state.getIn(['contexts', 'inReplyTos']), + ], (statusId, inReplyTos) => { let ancestorsIds = Immutable.List(); + ancestorsIds = ancestorsIds.withMutations(mutable => { + let id = statusId; + + while (id) { + mutable.unshift(id); + id = inReplyTos.get(id); + } + }); + + return ancestorsIds; + }); + + const getDescendantsIds = createSelector([ + (_, { id }) => id, + state => state.getIn(['contexts', 'replies']), + ], (statusId, contextReplies) => { let descendantsIds = Immutable.List(); + descendantsIds = descendantsIds.withMutations(mutable => { + const ids = [statusId]; - if (status) { - ancestorsIds = ancestorsIds.withMutations(mutable => { - let id = status.get('in_reply_to_id'); + while (ids.length > 0) { + let id = ids.shift(); + const replies = contextReplies.get(id); - while (id) { - mutable.unshift(id); - id = state.getIn(['contexts', 'inReplyTos', id]); + if (statusId !== id) { + mutable.push(id); } - }); - descendantsIds = descendantsIds.withMutations(mutable => { - const ids = [status.get('id')]; + if (replies) { + replies.reverse().forEach(reply => { + ids.unshift(reply); + }); + } + } + }); - while (ids.length > 0) { - let id = ids.shift(); - const replies = state.getIn(['contexts', 'replies', id]); + return descendantsIds; + }); - if (status.get('id') !== id) { - mutable.push(id); - } + const mapStateToProps = (state, props) => { + const status = getStatus(state, { id: props.params.statusId }); + let ancestorsIds = Immutable.List(); + let descendantsIds = Immutable.List(); - if (replies) { - replies.reverse().forEach(reply => { - ids.unshift(reply); - }); - } - } - }); + if (status) { + ancestorsIds = getAncestorsIds(state, { id: status.get('in_reply_to_id') }); + descendantsIds = getDescendantsIds(state, { id: status.get('id') }); } return {