Merge commit 'ed5491e5de6ede501715c421ad5fa53493f61250' into to_3.5.1

* commit 'ed5491e5de6ede501715c421ad5fa53493f61250': (45 commits)
  Bump version to 3.5.1 (mastodon#18000)
  Fix error resposes for `from` search prefix (mastodon#17963)
  Add v3.5.x to SECURITY.md (mastodon#17998)
  Fix dangling language-specific trends (mastodon#17997)
  Auto-fill timeline gaps when getting re-connecting to Websocket/EventSource stream (mastodon#17987)
  Fix extremely rare race condition when deleting a toot or account (mastodon#17994)
  Change e-mail notifications to only be sent when recipient is offline (mastodon#17984)
  Fix trends returning less results per page when filtered in REST API (mastodon#17996)
  Bump i18n-tasks from 0.9.37 to 1.0.8 (mastodon#17993)
  New Crowdin updates (mastodon#17897)
  Fix cookies secure flag being set when served over Tor (mastodon#17992)
  Bump gitlab-omniauth-openid-connect from 0.5.0 to 0.9.1 (mastodon#17779)
  Bump fabrication from 2.27.0 to 2.28.0 (mastodon#17960)
  Bump lograge from 0.11.2 to 0.12.0 (mastodon#17961)
  Fix migration error handling (mastodon#17991)
  Fix error re-running some migrations if they get interrupted at the wrong moment (mastodon#17989)
  Fix pagination header on empty trends responses in REST API (mastodon#17986)
  Fix potentially missing statuses when reconnecting to websocket (mastodon#17981)
  Fix failure when sending warning emails with custom text (mastodon#17983)
  Fix unset `SMTP_RETURN_PATH` environment variable causing e-mail not to send (mastodon#17982)
  ...

.eslintrc.js

@@ -79,6 +79,11 @@ module.exports = {
     'no-irregular-whitespace': 'error',
     'no-mixed-spaces-and-tabs': 'warn',
     'no-nested-ternary': 'warn',
+    'no-restricted-properties': [
+      'error',
+      { property: 'substring', message: 'Use .slice instead of .substring.' },
+      { property: 'substr', message: 'Use .slice instead of .substr.' },
+    ],
     'no-trailing-spaces': 'warn',
     'no-undef': 'error',
     'no-unreachable': 'error',

CHANGELOG.md

@@ -3,6 +3,49 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
+## [3.5.1] - 2022-04-08
+### Added
+
+- Add pagination for trending statuses in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/17976))
+
+### Changed
+
+- Change e-mail notifications to only be sent when recipient is offline ([Gargron](https://github.com/mastodon/mastodon/pull/17984))
+  - Send e-mails for mentions and follows by default again
+  - But only when recipient does not have push notifications through an app
+- Change `website` attribute to be nullable on `Application` entity in REST API ([rinsuki](https://github.com/mastodon/mastodon/pull/17962))
+
+### Removed
+
+- Remove sign-in token authentication, instead send e-mail about new sign-in ([Gargron](https://github.com/mastodon/mastodon/pull/17970))
+  - You no longer need to enter a security code sent through e-mail
+  - Instead you get an e-mail about a new sign-in from an unfamiliar IP address
+
+### Fixed
+
+- Fix error resposes for `from` search prefix ([single-right-quote](https://github.com/mastodon/mastodon/pull/17963))
+- Fix dangling language-specific trends ([Gargron](https://github.com/mastodon/mastodon/pull/17997))
+- Fix extremely rare race condition when deleting a status or account ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17994))
+- Fix trends returning less results per page when filtered in REST API ([Gargron](https://github.com/mastodon/mastodon/pull/17996))
+- Fix pagination header on empty trends responses in REST API ([Gargron](https://github.com/mastodon/mastodon/pull/17986))
+- Fix cookies secure flag being set when served over Tor ([Gargron](https://github.com/mastodon/mastodon/pull/17992))
+- Fix migration error handling ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17991))
+- Fix error when re-running some migrations if they get interrupted at the wrong moment ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17989))
+- Fix potentially missing statuses when reconnecting to streaming API in web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17981), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/17987), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/17980))
+- Fix error when sending warning emails with custom text ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17983))
+- Fix unset `SMTP_RETURN_PATH` environment variable causing e-mail not to send ([Gargron](https://github.com/mastodon/mastodon/pull/17982))
+- Fix possible duplicate statuses in timelines in some edge cases in web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17971))
+- Fix spurious edits and require incoming edits to be explicitly marked as such ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17918))
+- Fix error when encountering invalid pinned statuses ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17964))
+- Fix inconsistency in error handling when removing a status ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17974))
+- Fix admin API unconditionally requiring CSRF token ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17975))
+- Fix trending tags endpoint missing `offset` param in REST API ([Gargron](https://github.com/mastodon/mastodon/pull/17973))
+- Fix unusual number formatting in some locales ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17929))
+- Fix `S3_FORCE_SINGLE_REQUEST` environment variable not working ([HolgerHuo](https://github.com/mastodon/mastodon/pull/17922))
+- Fix failure to build assets with OpenSSL 3 ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17930))
+- Fix PWA manifest using outdated routes ([HolgerHuo](https://github.com/mastodon/mastodon/pull/17921))
+- Fix error when indexing statuses into Elasticsearch ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17912))
+
 ## [3.5.0] - 2022-03-30
 ### Added
 

Gemfile

@@ -40,7 +40,7 @@ end
 gem 'net-ldap', '~> 0.17'
 gem 'omniauth-cas', '~> 2.0'
 gem 'omniauth-saml', '~> 1.10'
-gem 'gitlab-omniauth-openid-connect', '~>0.5.0', require: 'omniauth_openid_connect'
+gem 'gitlab-omniauth-openid-connect', '~>0.9.1', require: 'omniauth_openid_connect'
 gem 'omniauth', '~> 1.9'
 gem 'omniauth-rails_csrf_protection', '~> 0.1'
 
@@ -99,9 +99,9 @@ gem 'json-ld-preloaded', '~> 3.2'
 gem 'rdf-normalize', '~> 0.5'
 
 group :development, :test do
-  gem 'fabrication', '~> 2.27'
+  gem 'fabrication', '~> 2.28'
   gem 'fuubar', '~> 2.5'
-  gem 'i18n-tasks', '~> 0.9', require: false
+  gem 'i18n-tasks', '~> 1.0', require: false
   gem 'pry-byebug', '~> 3.9'
   gem 'pry-rails', '~> 0.3'
   gem 'rspec-rails', '~> 5.1'
@@ -146,7 +146,7 @@ group :development do
 end
 
 group :production do
-  gem 'lograge', '~> 0.11'
+  gem 'lograge', '~> 0.12'
 end
 
 gem 'concurrent-ruby', require: false

Gemfile.lock

@@ -101,6 +101,14 @@ GEM
       coderay (>= 1.0.0)
       erubi (>= 1.0.0)
       rack (>= 0.9.0)
+    better_html (1.0.16)
+      actionview (>= 4.0)
+      activesupport (>= 4.0)
+      ast (~> 2.0)
+      erubi (~> 1.4)
+      html_tokenizer (~> 0.0.6)
+      parser (>= 2.4)
+      smart_properties
     bindata (2.4.10)
     binding_of_caller (1.0.0)
       debug_inspector (>= 0.0.1)
@@ -211,7 +219,7 @@ GEM
     et-orbi (1.2.6)
       tzinfo
     excon (0.76.0)
-    fabrication (2.27.0)
+    fabrication (2.28.0)
     faker (2.20.0)
       i18n (>= 1.8.11, < 2)
     faraday (1.9.3)
@@ -262,7 +270,7 @@ GEM
     fuubar (2.5.1)
       rspec-core (~> 3.0)
       ruby-progressbar (~> 1.4)
-    gitlab-omniauth-openid-connect (0.5.0)
+    gitlab-omniauth-openid-connect (0.9.1)
       addressable (~> 2.7)
       omniauth (~> 1.9)
       openid_connect (~> 1.2)
@@ -278,10 +286,11 @@ GEM
       hamlit (>= 1.2.0)
       railties (>= 4.0.1)
     hashdiff (1.0.1)
-    hashie (4.1.0)
+    hashie (5.0.0)
     highline (2.0.3)
     hiredis (0.6.3)
     hkdf (0.3.0)
+    html_tokenizer (0.0.7)
     htmlentities (4.3.4)
     http (5.0.4)
       addressable (~> 2.8)
@@ -298,9 +307,10 @@ GEM
       rainbow (>= 2.0.0)
     i18n (1.10.0)
       concurrent-ruby (~> 1.0)
-    i18n-tasks (0.9.37)
+    i18n-tasks (1.0.8)
       activesupport (>= 4.0.2)
       ast (>= 2.1.0)
+      better_html (~> 1.0)
       erubi
       highline (>= 2.0.0)
       i18n
@@ -360,12 +370,12 @@ GEM
     llhttp-ffi (0.4.0)
       ffi-compiler (~> 1.0)
       rake (~> 13.0)
-    lograge (0.11.2)
+    lograge (0.12.0)
       actionpack (>= 4)
       activesupport (>= 4)
       railties (>= 4)
       request_store (~> 1.0)
-    loofah (2.15.0)
+    loofah (2.16.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
@@ -417,7 +427,7 @@ GEM
     omniauth-saml (1.10.3)
       omniauth (~> 1.3, >= 1.3.2)
       ruby-saml (~> 1.9)
-    openid_connect (1.2.0)
+    openid_connect (1.3.0)
       activemodel
       attr_required (>= 1.0.0)
       json-jwt (>= 1.5.0)
@@ -430,14 +440,14 @@ GEM
     openssl (2.2.0)
     openssl-signature_algorithm (0.4.0)
     orm_adapter (0.5.0)
-    ox (2.14.10)
+    ox (2.14.11)
     parallel (1.22.1)
     parser (3.1.1.0)
       ast (~> 2.4.1)
     parslet (2.0.0)
     pastel (0.8.0)
       tty-color (~> 0.5)
-    pg (1.3.4)
+    pg (1.3.5)
     pghero (2.8.2)
       activerecord (>= 5)
     pkg-config (1.4.7)
@@ -459,7 +469,7 @@ GEM
     pry-rails (0.3.9)
       pry (>= 0.10.4)
     public_suffix (4.0.6)
-    puma (5.6.2)
+    puma (5.6.4)
       nio4r (~> 2.0)
     pundit (2.2.0)
       activesupport (>= 3.0.0)
@@ -470,7 +480,7 @@ GEM
       rack (>= 1.0, < 3)
     rack-cors (1.1.1)
       rack (>= 2.0.0)
-    rack-oauth2 (1.16.0)
+    rack-oauth2 (1.19.0)
       activesupport
       attr_required
       httpclient
@@ -525,7 +535,7 @@ GEM
     redis-namespace (1.8.2)
       redis (>= 3.0.4)
     regexp_parser (2.2.1)
-    request_store (1.5.0)
+    request_store (1.5.1)
       rack (>= 1.4)
     responders (3.0.1)
       actionpack (>= 5.0)
@@ -604,7 +614,7 @@ GEM
       sidekiq (>= 3)
       thwait
       tilt (>= 1.4.0)
-    sidekiq-unique-jobs (7.1.15)
+    sidekiq-unique-jobs (7.1.16)
       brpoplpush-redis_script (> 0.1.1, <= 2.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.5)
       sidekiq (>= 5.0, < 8.0)
@@ -620,6 +630,7 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.2)
+    smart_properties (1.17.0)
     sprockets (3.7.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
@@ -635,7 +646,7 @@ GEM
     stoplight (2.2.1)
     strong_migrations (0.7.9)
       activerecord (>= 5)
-    swd (1.2.0)
+    swd (1.3.0)
       activesupport (>= 3)
       attr_required (>= 0.0.5)
       httpclient (>= 2.4)
@@ -691,7 +702,7 @@ GEM
       safety_net_attestation (~> 0.4.0)
       securecompare (~> 1.0)
       tpm-key_attestation (~> 0.9.0)
-    webfinger (1.1.0)
+    webfinger (1.2.0)
       activesupport
       httpclient (>= 2.4)
     webmock (3.14.0)
@@ -750,21 +761,21 @@ DEPENDENCIES
   doorkeeper (~> 5.5)
   dotenv-rails (~> 2.7)
   ed25519 (~> 1.3)
-  fabrication (~> 2.27)
+  fabrication (~> 2.28)
   faker (~> 2.20)
   fast_blank (~> 1.0)
   fastimage
   fog-core (<= 2.1.0)
   fog-openstack (~> 0.3)
   fuubar (~> 2.5)
-  gitlab-omniauth-openid-connect (~> 0.5.0)
+  gitlab-omniauth-openid-connect (~> 0.9.1)
   hamlit-rails (~> 0.2)
   hiredis (~> 0.6)
   htmlentities (~> 4.3)
   http (~> 5.0)
   http_accept_language (~> 2.1)
   httplog (~> 1.5.0)
-  i18n-tasks (~> 0.9)
+  i18n-tasks (~> 1.0)
   idn-ruby
   json-ld
   json-ld-preloaded (~> 3.2)
@@ -773,7 +784,7 @@ DEPENDENCIES
   letter_opener (~> 1.8)
   letter_opener_web (~> 2.0)
   link_header (~> 0.0)
-  lograge (~> 0.11)
+  lograge (~> 0.12)
   makara (~> 0.5)
   mario-redis-lock (~> 1.2)
   memory_profiler

SECURITY.md

@@ -12,6 +12,7 @@ A "vulnerability in Mastodon" is a vulnerability in the code distributed through
 
 | Version | Supported          |
 | ------- | ------------------ |
+| 3.5.x   | Yes                |
 | 3.4.x   | Yes                |
 | 3.3.x   | Yes                |
 | < 3.3   | No                 |

app/chewy/statuses_index.rb

@@ -59,7 +59,7 @@ class StatusesIndex < Chewy::Index
     field :id, type: 'long'
     field :account_id, type: 'long'
 
-    field :text, type: 'text', value: ->(status) { [status.spoiler_text, extract_status_plain_text(status)].concat(status.ordered_media_attachments.map(&:description)).concat(status.preloadable_poll ? status.preloadable_poll.options : []).join("\n\n") } do
+    field :text, type: 'text', value: ->(status) { status.searchable_text } do
       field :stemmed, type: 'text', analyzer: 'content'
     end
 

app/controllers/api/v1/admin/account_actions_controller.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 class Api::V1::Admin::AccountActionsController < Api::BaseController
-  protect_from_forgery with: :exception
-
   before_action -> { authorize_if_got_token! :'admin:write', :'admin:write:accounts' }
   before_action :require_staff!
   before_action :set_account

app/controllers/api/v1/admin/accounts_controller.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 class Api::V1::Admin::AccountsController < Api::BaseController
-  protect_from_forgery with: :exception
-
   include Authorization
   include AccountableConcern
 
@@ -67,8 +65,9 @@ def reject
 
   def destroy
     authorize @account, :destroy?
+    json = render_to_body json: @account, serializer: REST::Admin::AccountSerializer
     Admin::AccountDeletionWorker.perform_async(@account.id)
-    render json: @account, serializer: REST::Admin::AccountSerializer
+    render json: json
   end
 
   def unsensitive

app/controllers/api/v1/admin/dimensions_controller.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 class Api::V1::Admin::DimensionsController < Api::BaseController
-  protect_from_forgery with: :exception
-
   before_action -> { authorize_if_got_token! :'admin:read' }
   before_action :require_staff!
   before_action :set_dimensions

app/controllers/api/v1/admin/measures_controller.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 class Api::V1::Admin::MeasuresController < Api::BaseController
-  protect_from_forgery with: :exception
-
   before_action -> { authorize_if_got_token! :'admin:read' }
   before_action :require_staff!
   before_action :set_measures

app/controllers/api/v1/admin/reports_controller.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 class Api::V1::Admin::ReportsController < Api::BaseController
-  protect_from_forgery with: :exception
-
   include Authorization
   include AccountableConcern
 

app/controllers/api/v1/admin/retention_controller.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 class Api::V1::Admin::RetentionController < Api::BaseController
-  protect_from_forgery with: :exception
-
   before_action -> { authorize_if_got_token! :'admin:read' }
   before_action :require_staff!
   before_action :set_cohorts