Merge tag 'v4.0.11' of https://github.com/mastodon/mastodon into HEAD

Co-authored-by: Claire <claire.github-309c@sitedethib.com>

.circleci/config.yml

@@ -1,223 +0,0 @@
-version: 2.1
-
-orbs:
-  ruby: circleci/ruby@1.4.1
-  node: circleci/node@5.0.1
-
-executors:
-  default:
-    parameters:
-      ruby-version:
-        type: string
-    docker:
-      - image: cimg/ruby:<< parameters.ruby-version >>
-        environment:
-          BUNDLE_JOBS: 3
-          BUNDLE_RETRY: 3
-          CONTINUOUS_INTEGRATION: true
-          DB_HOST: localhost
-          DB_USER: root
-          DISABLE_SIMPLECOV: true
-          RAILS_ENV: test
-      - image: cimg/postgres:14.0
-        environment:
-          POSTGRES_USER: root
-          POSTGRES_HOST_AUTH_METHOD: trust
-      - image: cimg/redis:6.2
-
-commands:
-  install-system-dependencies:
-    steps:
-      - run:
-          name: Install system dependencies
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y libicu-dev libidn11-dev
-  install-ruby-dependencies:
-    parameters:
-      ruby-version:
-        type: string
-    steps:
-      - run:
-          command: |
-            bundle config clean 'true'
-            bundle config frozen 'true'
-            bundle config without 'development production'
-          name: Set bundler settings
-      - ruby/install-deps:
-          bundler-version: '2.3.8'
-          key: ruby<< parameters.ruby-version >>-gems-v1
-  wait-db:
-    steps:
-      - run:
-          command: dockerize -wait tcp://localhost:5432 -wait tcp://localhost:6379 -timeout 1m
-          name: Wait for PostgreSQL and Redis
-
-jobs:
-  build:
-    docker:
-      - image: cimg/ruby:3.0-node
-        environment:
-          RAILS_ENV: test
-    steps:
-      - checkout
-      - install-system-dependencies
-      - install-ruby-dependencies:
-          ruby-version: '3.0'
-      - node/install-packages:
-          cache-version: v1
-          pkg-manager: yarn
-      - run:
-          command: ./bin/rails assets:precompile
-          name: Precompile assets
-      - persist_to_workspace:
-          paths:
-            - public/assets
-            - public/packs-test
-          root: .
-
-  test:
-    parameters:
-      ruby-version:
-        type: string
-    executor:
-      name: default
-      ruby-version: << parameters.ruby-version >>
-    environment:
-      ALLOW_NOPAM: true
-      PAM_ENABLED: true
-      PAM_DEFAULT_SERVICE: pam_test
-      PAM_CONTROLLED_SERVICE: pam_test_controlled
-    parallelism: 4
-    steps:
-      - checkout
-      - install-system-dependencies
-      - run:
-          command: sudo apt-get install -y ffmpeg imagemagick libpam-dev
-          name: Install additional system dependencies
-      - run:
-          command: bundle config with 'pam_authentication'
-          name: Enable PAM authentication
-      - install-ruby-dependencies:
-          ruby-version: << parameters.ruby-version >>
-      - attach_workspace:
-          at: .
-      - wait-db
-      - run:
-          command: ./bin/rails db:create db:schema:load db:seed
-          name: Load database schema
-      - ruby/rspec-test
-
-  test-migrations:
-    executor:
-      name: default
-      ruby-version: '3.0'
-    steps:
-      - checkout
-      - install-system-dependencies
-      - install-ruby-dependencies:
-          ruby-version: '3.0'
-      - wait-db
-      - run:
-          command: ./bin/rails db:create
-          name: Create database
-      - run:
-          command: ./bin/rails db:migrate VERSION=20171010025614
-          name: Run migrations up to v2.0.0
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate VERSION=20180514140000
-          name: Run migrations up to v2.4.0
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2_4
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate VERSION=20180707154237
-          name: Run migrations up to v2.4.3
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2_4_3
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate
-          name: Run all remaining migrations
-      - run:
-          command: ./bin/rails tests:migrations:check_database
-          name: Check migration result
-
-  test-two-step-migrations:
-    executor:
-      name: default
-      ruby-version: '3.0'
-    steps:
-      - checkout
-      - install-system-dependencies
-      - install-ruby-dependencies:
-          ruby-version: '3.0'
-      - wait-db
-      - run:
-          command: ./bin/rails db:create
-          name: Create database
-      - run:
-          command: ./bin/rails db:migrate VERSION=20171010025614
-          name: Run migrations up to v2.0.0
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate VERSION=20180514140000
-          name: Run pre-deployment migrations up to v2.4.0
-          environment:
-            SKIP_POST_DEPLOYMENT_MIGRATIONS: true
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2_4
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate VERSION=20180707154237
-          name: Run migrations up to v2.4.3
-          environment:
-            SKIP_POST_DEPLOYMENT_MIGRATIONS: true
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2_4_3
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate
-          name: Run all remaining pre-deployment migrations
-          environment:
-            SKIP_POST_DEPLOYMENT_MIGRATIONS: true
-      - run:
-          command: ./bin/rails db:migrate
-          name: Run all post-deployment migrations
-      - run:
-          command: ./bin/rails tests:migrations:check_database
-          name: Check migration result
-
-workflows:
-  version: 2
-  build-and-test:
-    jobs:
-      - build
-      - test:
-          matrix:
-            parameters:
-              ruby-version:
-                - '2.7'
-                - '3.0'
-          name: test-ruby<< matrix.ruby-version >>
-          requires:
-            - build
-      - test-migrations:
-          requires:
-            - build
-      - test-two-step-migrations:
-          requires:
-            - build
-      - node/run:
-          cache-version: v1
-          name: test-webui
-          pkg-manager: yarn
-          requires:
-            - build
-          version: lts
-          yarn-run: test:jest

.github/workflows/build-container-image.yml

@@ -0,0 +1,92 @@
+on:
+  workflow_call:
+    inputs:
+      platforms:
+        required: true
+        type: string
+      cache:
+        type: boolean
+        default: true
+      use_native_arm64_builder:
+        type: boolean
+      push_to_images:
+        type: string
+      flavor:
+        type: string
+      tags:
+        type: string
+      labels:
+        type: string
+
+jobs:
+  build-image:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: docker/setup-qemu-action@v2
+        if: contains(inputs.platforms, 'linux/arm64') && !inputs.use_native_arm64_builder
+
+      - uses: docker/setup-buildx-action@v2
+        id: buildx
+        if: ${{ !(inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64')) }}
+
+      - name: Start a local Docker Builder
+        if: inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64')
+        run: |
+          docker run --rm -d --name buildkitd -p 1234:1234 --privileged moby/buildkit:latest --addr tcp://0.0.0.0:1234
+
+      - uses: docker/setup-buildx-action@v2
+        id: buildx-native
+        if: inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64')
+        with:
+          driver: remote
+          endpoint: tcp://localhost:1234
+          platforms: linux/amd64
+          append: |
+            - endpoint: tcp://${{ vars.DOCKER_BUILDER_HETZNER_ARM64_01_HOST }}:13865
+              platforms: linux/arm64
+              name: mastodon-docker-builder-arm64-01
+              driver-opts:
+                - servername=mastodon-docker-builder-arm64-01
+        env:
+          BUILDER_NODE_1_AUTH_TLS_CACERT: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_CACERT }}
+          BUILDER_NODE_1_AUTH_TLS_CERT: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_CERT }}
+          BUILDER_NODE_1_AUTH_TLS_KEY: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_KEY }}
+
+      - name: Log in to Docker Hub
+        if: contains(inputs.push_to_images, 'tootsuite')
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Log in to the Github Container registry
+        if: contains(inputs.push_to_images, 'ghcr.io')
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: docker/metadata-action@v4
+        id: meta
+        if: ${{ inputs.push_to_images != '' }}
+        with:
+          images: ${{ inputs.push_to_images }}
+          flavor: ${{ inputs.flavor }}
+          tags: ${{ inputs.tags }}
+          labels: ${{ inputs.labels }}
+
+      - uses: docker/build-push-action@v4
+        with:
+          context: .
+          platforms: ${{ inputs.platforms }}
+          provenance: false
+          builder: ${{ steps.buildx.outputs.name || steps.buildx-native.outputs.name }}
+          push: ${{ inputs.push_to_images != '' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: ${{ inputs.cache && 'type=gha' || '' }}
+          cache-to: ${{ inputs.cache && 'type=gha,mode=max' || '' }}

.github/workflows/build-image.yml

@@ -1,46 +0,0 @@
-name: Build container image
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - 'main'
-    tags:
-      - '*'
-  pull_request:
-    paths:
-      - .github/workflows/build-image.yml
-      - Dockerfile
-permissions:
-  contents: read
-
-jobs:
-  build-image:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-qemu-action@v2
-      - uses: docker/setup-buildx-action@v2
-      - uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-        if: github.event_name != 'pull_request'
-      - uses: docker/metadata-action@v4
-        id: meta
-        with:
-          images: tootsuite/mastodon
-          flavor: |
-            latest=auto
-          tags: |
-            type=edge,branch=main
-            type=pep440,pattern={{raw}}
-            type=pep440,pattern=v{{major}}.{{minor}}
-            type=ref,event=pr
-      - uses: docker/build-push-action@v3
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          cache-from: type=registry,ref=tootsuite/mastodon:edge
-          cache-to: type=inline

.github/workflows/build-releases.yml

@@ -0,0 +1,27 @@
+name: Build container release images
+on:
+  push:
+    tags:
+      - '*'
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build-image:
+    uses: ./.github/workflows/build-container-image.yml
+    with:
+      platforms: linux/amd64,linux/arm64
+      use_native_arm64_builder: true
+      push_to_images: |
+        tootsuite/mastodon
+        ghcr.io/mastodon/mastodon
+      # Do not use cache when building releases, so apt update is always ran and the release always contain the latest packages
+      cache: false
+      flavor: |
+        latest=false
+      tags: |
+        type=pep440,pattern={{raw}}
+        type=pep440,pattern=v{{major}}.{{minor}}
+    secrets: inherit

.github/workflows/test-image-build.yml

@@ -0,0 +1,15 @@
+name: Test container image build
+on:
+  pull_request:
+permissions:
+  contents: read
+
+jobs:
+  build-image:
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+
+    uses: ./.github/workflows/build-container-image.yml
+    with:
+      platforms: linux/amd64 # Testing only on native platform so it is performant

.ruby-version

@@ -1 +1 @@
-3.0.4
+3.0.6

CHANGELOG.md

@@ -3,6 +3,166 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
+## End of life notice
+
+**The 4.0.x branch will not receive any update after 2023-10-31.**
+This means that no security fix will be made available for this branch after this date, and you will need to update to a more recent version (such as the 4.1.x branch) to receive security fixes.
+
+## [4.0.11] - 2023-09-20
+
+### Fixed
+
+- Fix post translation erroring out ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26990))
+
+## [4.0.10] - 2023-09-19
+
+### Fixed
+
+- Fix moderator rights inconsistencies ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26729))
+- Fix crash when encountering invalid URL ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26814))
+- Fix cached posts including stale stats ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26409))
+- Fix uploading of video files for which `ffprobe` reports `0/0` average framerate ([NicolaiSoeborg](https://github.com/mastodon/mastodon/pull/26500))
+- Fix unexpected audio stream transcoding when uploaded video is eligible to passthrough ([yufushiro](https://github.com/mastodon/mastodon/pull/26608))
+
+### Security
+
+- Fix missing HTML sanitization in translation API (CVE-2023-42452)
+- Fix incorrect domain name normalization (CVE-2023-42451)
+
+## [4.0.9] - 2023-09-05
+
+### Changed
+
+- Change remote report processing to accept reports with long comments, but truncate them ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25028))
+
+### Fixed
+
+- **Fix blocking subdomains of an already-blocked domain** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26392))
+- Fix `/api/v1/timelines/tag/:hashtag` allowing for unauthenticated access when public preview is disabled ([danielmbrasil](https://github.com/mastodon/mastodon/pull/26237))
+- Fix inefficiencies in `PlainTextFormatter` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26727))
+
+## [4.0.8] - 2023-07-31
+
+### Fixed
+
+- Fix memory leak in streaming server ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/26228))
+- Fix wrong filters sometimes applying in streaming ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26159), [ThisIsMissEm](https://github.com/mastodon/mastodon/pull/26213), [renchap](https://github.com/mastodon/mastodon/pull/26233))
+- Fix incorrect connect timeout in outgoing requests ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26116))
+
+## [4.0.7] - 2023-07-21
+
+### Added
+
+- Add check preventing Sidekiq workers from running with Makara configured ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25850))
+
+### Changed
+
+- Change request timeout handling to use a longer deadline ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26055))
+
+### Fixed
+
+- Fix moderation interface for remote instances with a .zip TLD ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25886))
+- Fix remote accounts being possibly persisted to database with incomplete protocol values ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25886))
+- Fix trending publishers table not rendering correctly on narrow screens ([vmstan](https://github.com/mastodon/mastodon/pull/25945))
+
+### Security
+
+- Fix CSP headers being unintentionally wide ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26105))
+
+## [4.0.6] - 2023-07-07
+
+### Fixed
+
+- Fix branding:generate_app_icons failing because of disallowed ICO coder ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25794))
+- Fix crash in admin interface when viewing a remote user with verified links ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25796))
+- Fix processing of media files with unusual names ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25788))
+
+## [4.0.5] - 2023-07-06
+
+### Changed
+
+- Change OpenGraph-based embeds to allow fullscreen ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25058))
+- Change profile updates to be sent to recently-mentioned servers ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24852))
+- Change `/api/v1/statuses/:id/history` to always return at least one item ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25510))
+- Change auto-linking to allow carets in URL query params ([renchap](https://github.com/mastodon/mastodon/pull/25216))
+
+### Removed
+
+- Remove invalid `X-Frame-Options: ALLOWALL` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25070))
+
+### Fixed
+
+- Fix wrong view being displayed when a webhook fails validation ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25464))
+- Fix soft-deleted post cleanup scheduler overwhelming the streaming server ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25519))
+- Fix incorrect pagination headers in `/api/v2/admin/accounts` ([danielmbrasil](https://github.com/mastodon/mastodon/pull/25477))
+- Fix performance of streaming by parsing message JSON once ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25278), [ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25361))
+- Fix CSP headers when `S3_ALIAS_HOST` includes a path component ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25273))
+- Fix `tootctl accounts approve --number N` not aproving N earliest registrations ([danielmbrasil](https://github.com/mastodon/mastodon/pull/24605))
+- Fix being able to vote on your own polls ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25015))
+- Fix race condition when reblogging a status ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25016))
+- Fix “Authorized applications” inefficiently and incorrectly getting last use date ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25060))
+- Fix multiple N+1s in ConversationsController ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25134), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25399), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25499))
+- Fix user archive takeouts when using OpenStack Swift ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24431))
+- Fix searching for remote content by URL not working under certain conditions ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25637))
+- Fix inefficiencies in indexing content for search ([VyrCossont](https://github.com/mastodon/mastodon/pull/24285), [VyrCossont](https://github.com/mastodon/mastodon/pull/24342))
+
+### Security
+
+- Add finer permission requirements for managing webhooks ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25463))
+- Update dependencies
+- Add hardening headers for user-uploaded files ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25756))
+- Fix verified links possibly hiding important parts of the URL (CVE-2023-36462)
+- Fix timeout handling of outbound HTTP requests (CVE-2023-36461)
+- Fix arbitrary file creation through media processing (CVE-2023-36460)
+- Fix possible XSS in preview cards (CVE-2023-36459)
+
+## [4.0.4] - 2023-04-04
+
+### Fixed
+
+- Fix crash in `tootctl` commands making use of parallelization when Elasticsearch is enabled ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24182), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24377))
+- Fix crash in `db:setup` when Elasticsearch is enabled ([rrgeorge](https://github.com/mastodon/mastodon/pull/24302))
+- Fix user archive takeout when using OpenStack Swift or S3 providers with no ACL support ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24200))
+- Fix invalid/expired invites being processed on sign-up ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24337))
+
+### Security
+
+- Update Ruby to 3.0.6 due to ReDoS vulnerabilities ([saizai](https://github.com/mastodon/mastodon/pull/24333))
+- Fix unescaped user input in LDAP query ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24379))
+
+# [4.0.3] - 2023-03-16
+
+### Added
+
+- Add redirection from paths with url-encoded `@` to their decoded form ([thijskh](https://github.com/mastodon/mastodon/pull/23593))
+- Add `lang` attribute to native language names in language picker in Web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23749))
+- Add headers to outgoing mails to avoid auto-replies ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23597))
+
+### Fixed
+
+- Fix “Remove all followers from the selected domains” being more destructive than it claims ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23805))
+- Fix case-sensitive check for previously used hashtags in hashtag autocompletion ([deanveloper](https://github.com/mastodon/mastodon/pull/23526))
+- Fix sidebar behavior in settings/admin UI on mobile ([wxt2005](https://github.com/mastodon/mastodon/pull/23764))
+- Fix inefficiency when searching accounts per username in admin interface ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23801))
+- Fix server error when failing to follow back followers from `/relationships` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23787))
+- Fix server error when attempting to display the edit history of a trendable post in the admin interface ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23574))
+- Fix original account being unfollowed on migration before the follow request to the new account could be sent ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/21957))
+- Fix pgBouncer resetting application name on every transaction ([Gargron](https://github.com/mastodon/mastodon/pull/23958))
+- Fix unconfirmed accounts being counted as active users ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23803))
+- Fix `/api/v1/streaming` sub-paths not being redirected ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23988))
+- Fix drag'n'drop upload area text that spans multiple lines not being centered ([vintprox](https://github.com/mastodon/mastodon/pull/24029))
+- Fix sidekiq jobs not triggering Elasticsearch index updates ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24046))
+- Fix tags being unnecessarily stripped from plain-text short site description ([c960657](https://github.com/mastodon/mastodon/pull/23975))
+- Fix HTML entities not being un-escaped in extracted plain-text from remote posts ([c960657](https://github.com/mastodon/mastodon/pull/24019))
+- Fix dashboard crash on ElasticSearch server error ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23751))
+- Fix incorrect post links in strikes when the account is remote ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23611))
+- Fix misleading error code when receiving invalid WebAuthn credentials ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23568))
+
+### Security
+
+- Change user backups to use expiring URLs for download when possible ([Gargron](https://github.com/mastodon/mastodon/pull/24136))
+- Add warning for object storage misconfiguration ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24137))
+
 ## [4.0.2] - 2022-11-15
 ### Fixed
 

Dockerfile

@@ -19,6 +19,7 @@ RUN ARCH= && \
   esac && \
     echo "Etc/UTC" > /etc/localtime && \
 	apt-get update && \
+	apt-get -yq dist-upgrade && \
 	apt-get install -y --no-install-recommends ca-certificates wget python3 apt-utils && \
 	cd ~ && \
 	wget -q https://nodejs.org/download/release/v$NODE_VER/node-v$NODE_VER-linux-$ARCH.tar.gz && \
@@ -27,7 +28,7 @@ RUN ARCH= && \
 	mv node-v$NODE_VER-linux-$ARCH /opt/node
 
 # Install Ruby 3.0
-ENV RUBY_VER="3.0.4"
+ENV RUBY_VER="3.0.6"
 RUN apt-get update && \
   apt-get install -y --no-install-recommends build-essential \
     bison libyaml-dev libgdbm-dev libreadline-dev libjemalloc-dev \
@@ -46,7 +47,7 @@ RUN apt-get update && \
 
 ENV PATH="${PATH}:/opt/ruby/bin:/opt/node/bin"
 
-RUN npm install -g npm@latest && \
+RUN npm install -g npm@9 && \
 	npm install -g yarn && \
 	gem install bundler && \
 	apt-get update && \

Gemfile.lock

@@ -10,40 +10,40 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (6.1.7)
-      actionpack (= 6.1.7)
-      activesupport (= 6.1.7)
+    actioncable (6.1.7.4)
+      actionpack (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.1.7)
-      actionpack (= 6.1.7)
-      activejob (= 6.1.7)
-      activerecord (= 6.1.7)
-      activestorage (= 6.1.7)
-      activesupport (= 6.1.7)
+    actionmailbox (6.1.7.4)
+      actionpack (= 6.1.7.4)
+      activejob (= 6.1.7.4)
+      activerecord (= 6.1.7.4)
+      activestorage (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
       mail (>= 2.7.1)
-    actionmailer (6.1.7)
-      actionpack (= 6.1.7)
-      actionview (= 6.1.7)
-      activejob (= 6.1.7)
-      activesupport (= 6.1.7)
+    actionmailer (6.1.7.4)
+      actionpack (= 6.1.7.4)
+      actionview (= 6.1.7.4)
+      activejob (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.1.7)
-      actionview (= 6.1.7)
-      activesupport (= 6.1.7)
+    actionpack (6.1.7.4)
+      actionview (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
       rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.1.7)
-      actionpack (= 6.1.7)
-      activerecord (= 6.1.7)
-      activestorage (= 6.1.7)
-      activesupport (= 6.1.7)
+    actiontext (6.1.7.4)
+      actionpack (= 6.1.7.4)
+      activerecord (= 6.1.7.4)
+      activestorage (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
       nokogiri (>= 1.8.5)
-    actionview (6.1.7)
-      activesupport (= 6.1.7)
+    actionview (6.1.7.4)
+      activesupport (= 6.1.7.4)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
@@ -54,22 +54,22 @@ GEM
       case_transform (>= 0.2)
       jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
     active_record_query_trace (1.8)
-    activejob (6.1.7)
-      activesupport (= 6.1.7)
+    activejob (6.1.7.4)
+      activesupport (= 6.1.7.4)
       globalid (>= 0.3.6)
-    activemodel (6.1.7)
-      activesupport (= 6.1.7)
-    activerecord (6.1.7)
-      activemodel (= 6.1.7)
-      activesupport (= 6.1.7)
-    activestorage (6.1.7)
-      actionpack (= 6.1.7)
-      activejob (= 6.1.7)
-      activerecord (= 6.1.7)
-      activesupport (= 6.1.7)
+    activemodel (6.1.7.4)
+      activesupport (= 6.1.7.4)
+    activerecord (6.1.7.4)
+      activemodel (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
+    activestorage (6.1.7.4)
+      actionpack (= 6.1.7.4)
+      activejob (= 6.1.7.4)
+      activerecord (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
       marcel (~> 1.0)
       mini_mime (>= 1.1.0)
-    activesupport (6.1.7)
+    activesupport (6.1.7.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -206,7 +206,7 @@ GEM
     docile (1.3.4)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    doorkeeper (5.6.0)
+    doorkeeper (5.6.6)
       railties (>= 5)
     dotenv (2.8.1)
     dotenv-rails (2.8.1)
@@ -282,7 +282,7 @@ GEM
       addressable (~> 2.7)
       omniauth (>= 1.9, < 3)
       openid_connect (~> 1.2)
-    globalid (1.0.0)
+    globalid (1.0.1)
       activesupport (>= 5.0)
     hamlit (2.13.0)
       temple (>= 0.8.2)
@@ -382,7 +382,7 @@ GEM
       activesupport (>= 4)
       railties (>= 4)
       request_store (~> 1.0)
-    loofah (2.19.0)
+    loofah (2.19.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
@@ -402,7 +402,7 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2022.0105)
     mini_mime (1.1.2)
-    mini_portile2 (2.8.0)
+    mini_portile2 (2.8.2)
     minitest (5.16.3)
     msgpack (1.5.4)
     multi_json (1.15.0)
@@ -411,9 +411,9 @@ GEM
     net-scp (4.0.0.rc1)
       net-ssh (>= 2.6.5, < 8.0.0)
     net-ssh (7.0.1)
-    nio4r (2.5.8)
-    nokogiri (1.13.9)
-      mini_portile2 (~> 2.8.0)
+    nio4r (2.5.9)
+    nokogiri (1.15.3)
+      mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     nsa (0.2.8)
       activesupport (>= 4.2, < 7)
@@ -482,8 +482,8 @@ GEM
     pundit (2.2.0)
       activesupport (>= 3.0.0)
     raabro (1.4.0)
-    racc (1.6.0)
-    rack (2.2.4)
+    racc (1.7.1)
+    rack (2.2.7)
     rack-attack (6.6.1)
       rack (>= 1.0, < 3)
     rack-cors (1.1.1)
@@ -498,20 +498,20 @@ GEM
       rack
     rack-test (2.0.2)
       rack (>= 1.3)
-    rails (6.1.7)
-      actioncable (= 6.1.7)
-      actionmailbox (= 6.1.7)
-      actionmailer (= 6.1.7)
-      actionpack (= 6.1.7)
-      actiontext (= 6.1.7)
-      actionview (= 6.1.7)
-      activejob (= 6.1.7)
-      activemodel (= 6.1.7)
-      activerecord (= 6.1.7)
-      activestorage (= 6.1.7)
-      activesupport (= 6.1.7)
+    rails (6.1.7.4)
+      actioncable (= 6.1.7.4)
+      actionmailbox (= 6.1.7.4)
+      actionmailer (= 6.1.7.4)
+      actionpack (= 6.1.7.4)
+      actiontext (= 6.1.7.4)
+      actionview (= 6.1.7.4)
+      activejob (= 6.1.7.4)
+      activemodel (= 6.1.7.4)
+      activerecord (= 6.1.7.4)
+      activestorage (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
       bundler (>= 1.15.0)
-      railties (= 6.1.7)
+      railties (= 6.1.7.4)
       sprockets-rails (>= 2.0.0)
     rails-controller-testing (1.0.5)
       actionpack (>= 5.0.1.rc1)
@@ -520,16 +520,16 @@ GEM
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.4.3)
-      loofah (~> 2.3)
+    rails-html-sanitizer (1.4.4)
+      loofah (~> 2.19, >= 2.19.1)
     rails-i18n (6.0.0)
       i18n (>= 0.7, < 2)
       railties (>= 6.0.0, < 7)
     rails-settings-cached (0.6.6)
       rails (>= 4.2.0)
-    railties (6.1.7)
-      actionpack (= 6.1.7)
-      activesupport (= 6.1.7)
+    railties (6.1.7.4)
+      actionpack (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
@@ -602,7 +602,7 @@ GEM
       fugit (~> 1.1, >= 1.1.6)
     safety_net_attestation (0.4.0)
       jwt (~> 2.0)
-    sanitize (6.0.0)
+    sanitize (6.0.2)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     scenic (1.6.0)
@@ -661,7 +661,7 @@ GEM
       unicode-display_width (>= 1.1.1, < 3)
     terrapin (0.6.0)
       climate_control (>= 0.0.3, < 1.0)
-    thor (1.2.1)
+    thor (1.2.2)
     tilt (2.0.11)
     tpm-key_attestation (0.11.0)
       bindata (~> 2.4)
@@ -680,7 +680,7 @@ GEM
     twitter-text (3.1.0)
       idn-ruby
       unf (~> 0.1.0)
-    tzinfo (2.0.5)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     tzinfo-data (1.2022.4)
       tzinfo (>= 1.0.0)
@@ -725,7 +725,7 @@ GEM
     xorcist (1.1.3)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.6.0)
+    zeitwerk (2.6.8)
 
 PLATFORMS
   ruby

README.md

@@ -8,13 +8,11 @@
 [![Build Status](https://img.shields.io/circleci/project/github/mastodon/mastodon.svg)][circleci]
 [![Code Climate](https://img.shields.io/codeclimate/maintainability/mastodon/mastodon.svg)][code_climate]
 [![Crowdin](https://d322cqt584bo4o.cloudfront.net/mastodon/localized.svg)][crowdin]
-[![Docker Pulls](https://img.shields.io/docker/pulls/tootsuite/mastodon.svg)][docker]
 
 [releases]: https://github.com/mastodon/mastodon/releases
 [circleci]: https://circleci.com/gh/mastodon/mastodon
 [code_climate]: https://codeclimate.com/github/mastodon/mastodon
 [crowdin]: https://crowdin.com/project/mastodon
-[docker]: https://hub.docker.com/r/tootsuite/mastodon/
 
 Mastodon is a **free, open-source social network server** based on ActivityPub where users can follow friends and discover new ones. On Mastodon, users can publish anything they want: links, pictures, text, video. All Mastodon servers are interoperable as a federated network (users on one server can seamlessly communicate with users from another one, including non-Mastodon software that implements ActivityPub)!
 
@@ -31,6 +29,7 @@ Click below to **learn more** in a video:
 - [View sponsors](https://joinmastodon.org/sponsors)
 - [Blog](https://blog.joinmastodon.org)
 - [Documentation](https://docs.joinmastodon.org)
+- [Official Docker image](https://github.com/mastodon/mastodon/pkgs/container/mastodon)
 - [Browse Mastodon servers](https://joinmastodon.org/communities)
 - [Browse Mastodon apps](https://joinmastodon.org/apps)
 

SECURITY.md

@@ -10,8 +10,9 @@ A "vulnerability in Mastodon" is a vulnerability in the code distributed through
 
 ## Supported Versions
 
-| Version | Supported |
-| ------- | ----------|
-| 4.0.x   | Yes       |
-| 3.5.x   | Yes       |
-| < 3.5   | No        |
+| Version | Supported          |
+| ------- | ------------------ |
+| 4.1.x   | Yes                |
+| 4.0.x   | Until 2023-10-31   |
+| 3.5.x   | Until 2023-12-31   |
+| < 3.5   | No                 |

app/controllers/admin/domain_blocks_controller.rb

@@ -25,7 +25,7 @@ module Admin
         @domain_block.errors.delete(:domain)
         render :new
       else
-        if existing_domain_block.present?
+        if existing_domain_block.present? && existing_domain_block.domain == TagManager.instance.normalize_domain(@domain_block.domain.strip)
           @domain_block = existing_domain_block
           @domain_block.update(resource_params)
         end

app/controllers/admin/webhooks_controller.rb

@@ -20,6 +20,7 @@ module Admin
       authorize :webhook, :create?
 
       @webhook = Webhook.new(resource_params)
+      @webhook.current_account = current_account
 
       if @webhook.save
         redirect_to admin_webhook_path(@webhook)
@@ -39,10 +40,12 @@ module Admin
     def update
       authorize @webhook, :update?
 
+      @webhook.current_account = current_account
+
       if @webhook.update(resource_params)
         redirect_to admin_webhook_path(@webhook)
       else
-        render :show
+        render :edit
       end
     end
 

app/controllers/api/v1/conversations_controller.rb

@@ -11,7 +11,7 @@ class Api::V1::ConversationsController < Api::BaseController
 
   def index
     @conversations = paginated_conversations
-    render json: @conversations, each_serializer: REST::ConversationSerializer
+    render json: @conversations, each_serializer: REST::ConversationSerializer, relationships: StatusRelationshipsPresenter.new(@conversations.map(&:last_status), current_user&.account_id)
   end
 
   def read
@@ -32,6 +32,19 @@ class Api::V1::ConversationsController < Api::BaseController
 
   def paginated_conversations
     AccountConversation.where(account: current_account)
+                       .includes(
+                         account: :account_stat,
+                         last_status: [
+                           :media_attachments,
+                           :preview_cards,
+                           :status_stat,
+                           :tags,
+                           {
+                             active_mentions: [account: :account_stat],
+                             account: :account_stat,
+                           },
+                         ]
+                       )
                        .to_a_paginated_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id))
   end
 

app/controllers/api/v1/statuses/histories_controller.rb

@@ -7,11 +7,15 @@ class Api::V1::Statuses::HistoriesController < Api::BaseController
   before_action :set_status
 
   def show
-    render json: @status.edits.includes(:account, status: [:account]), each_serializer: REST::StatusEditSerializer
+    render json: status_edits, each_serializer: REST::StatusEditSerializer
   end
 
   private
 
+  def status_edits
+    @status.edits.includes(:account, status: [:account]).to_a.presence || [@status.build_snapshot(at_time: @status.edited_at || @status.created_at)]
+  end
+
   def set_status
     @status = Status.find(params[:status_id])
     authorize @status, :show?

app/controllers/api/v1/statuses/reblogs_controller.rb

@@ -2,6 +2,8 @@
 
 class Api::V1::Statuses::ReblogsController < Api::BaseController
   include Authorization
+  include Redisable
+  include Lockable
 
   before_action -> { doorkeeper_authorize! :write, :'write:statuses' }
   before_action :require_user!
@@ -10,7 +12,9 @@ class Api::V1::Statuses::ReblogsController < Api::BaseController
   override_rate_limit_headers :create, family: :statuses
 
   def create
-    @status = ReblogService.new.call(current_account, @reblog, reblog_params)
+    with_lock("reblog:#{current_account.id}:#{@reblog.id}") do
+      @status = ReblogService.new.call(current_account, @reblog, reblog_params)
+    end
 
     render json: @status, serializer: REST::StatusSerializer
   end

app/controllers/api/v1/timelines/tag_controller.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 class Api::V1::Timelines::TagController < Api::BaseController
+  before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, only: :show, if: :require_auth?
   before_action :load_tag
   after_action :insert_pagination_headers, unless: -> { @statuses.empty? }
 
@@ -11,6 +12,10 @@ class Api::V1::Timelines::TagController < Api::BaseController
 
   private
 
+  def require_auth?
+    !Setting.timeline_preview
+  end
+
   def load_tag
     @tag = Tag.find_normalized(params[:id])
   end

app/controllers/api/v2/admin/accounts_controller.rb

@@ -18,6 +18,14 @@ class Api::V2::Admin::AccountsController < Api::V1::Admin::AccountsController
 
   private
 
+  def next_path
+    api_v2_admin_accounts_url(pagination_params(max_id: pagination_max_id)) if records_continue?
+  end
+
+  def prev_path
+    api_v2_admin_accounts_url(pagination_params(min_id: pagination_since_id)) unless @accounts.empty?
+  end
+
   def filtered_accounts
     AccountFilter.new(translated_filter_params).results
   end

app/controllers/auth/registrations_controller.rb

@@ -48,7 +48,7 @@ class Auth::RegistrationsController < Devise::RegistrationsController
     super(hash)
 
     resource.locale                 = I18n.locale
-    resource.invite_code            = params[:invite_code] if resource.invite_code.blank?
+    resource.invite_code            = @invite&.code if resource.invite_code.blank?
     resource.registration_form_time = session[:registration_form_time]
     resource.sign_up_ip             = request.remote_ip
 

app/controllers/backups_controller.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class BackupsController < ApplicationController
+  include RoutingHelper
+
+  skip_before_action :require_functional!
+
+  before_action :authenticate_user!
+  before_action :set_backup
+
+  def download
+    case Paperclip::Attachment.default_options[:storage]
+    when :s3
+      redirect_to @backup.dump.expiring_url(10)
+    when :fog
+      if Paperclip::Attachment.default_options.dig(:fog_credentials, :openstack_temp_url_key).present?
+        redirect_to @backup.dump.expiring_url(Time.now.utc + 10)
+      else
+        redirect_to full_asset_url(@backup.dump.url)
+      end
+    when :filesystem
+      redirect_to full_asset_url(@backup.dump.url)
+    end
+  end
+
+  private
+
+  def set_backup
+    @backup = current_user.backups.find(params[:id])
+  end
+end

app/controllers/media_controller.rb

@@ -46,6 +46,6 @@ class MediaController < ApplicationController
   end
 
   def allow_iframing
-    response.headers['X-Frame-Options'] = 'ALLOWALL'
+    response.headers.delete('X-Frame-Options')
   end
 end

app/controllers/oauth/authorized_applications_controller.rb

@@ -8,6 +8,8 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio
   before_action :require_not_suspended!, only: :destroy
   before_action :set_body_classes
 
+  before_action :set_last_used_at_by_app, only: :index, unless: -> { request.format == :json }
+
   skip_before_action :require_functional!
 
   include Localized
@@ -30,4 +32,14 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio
   def require_not_suspended!
     forbidden if current_account.suspended?
   end
+
+  def set_last_used_at_by_app
+    @last_used_at_by_app = Doorkeeper::AccessToken
+                           .select('DISTINCT ON (application_id) application_id, last_used_at')
+                           .where(resource_owner_id: current_resource_owner.id)
+                           .where.not(last_used_at: nil)
+                           .order(application_id: :desc, last_used_at: :desc)
+                           .pluck(:application_id, :last_used_at)
+                           .to_h
+  end
 end

app/controllers/relationships_controller.rb

@@ -19,6 +19,8 @@ class RelationshipsController < ApplicationController
     @form.save
   rescue ActionController::ParameterMissing
     # Do nothing
+  rescue Mastodon::NotPermittedError, ActiveRecord::RecordNotFound
+    flash[:alert] = I18n.t('relationships.follow_failure') if action_from_button == 'follow'
   ensure
     redirect_to relationships_path(filter_params)
   end
@@ -60,8 +62,8 @@ class RelationshipsController < ApplicationController
       'unfollow'
     elsif params[:remove_from_followers]
       'remove_from_followers'
-    elsif params[:block_domains]
-      'block_domains'
+    elsif params[:block_domains] || params[:remove_domains_from_followers]
+      'remove_domains_from_followers'
     end
   end
 

app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb

@@ -52,7 +52,7 @@ module Settings
             end
           else
             flash[:error] = I18n.t('webauthn_credentials.create.error')
-            status = :internal_server_error
+            status = :unprocessable_entity
           end
         else
           flash[:error] = t('webauthn_credentials.create.error')

app/controllers/statuses_controller.rb

@@ -43,7 +43,7 @@ class StatusesController < ApplicationController
     return not_found if @status.hidden? || @status.reblog?
 
     expires_in 180, public: true
-    response.headers['X-Frame-Options'] = 'ALLOWALL'
+    response.headers.delete('X-Frame-Options')
 
     render layout: 'embedded'
   end

app/helpers/formatting_helper.rb

@@ -49,6 +49,10 @@ module FormattingHelper
   end
 
   def account_field_value_format(field, with_rel_me: true)
-    html_aware_format(field.value, field.account.local?, with_rel_me: with_rel_me, with_domains: true, multiline: false)
+    if field.verified? && !field.account.local?
+      TextFormatter.shortened_link(field.value_for_verification)
+    else
+      html_aware_format(field.value, field.account.local?, with_rel_me: with_rel_me, with_domains: true, multiline: false)
+    end
   end
 end

app/javascript/mastodon/features/compose/components/language_dropdown.js

@@ -216,7 +216,7 @@ class LanguageDropdownMenu extends React.PureComponent {
 
     return (
       <div key={lang[0]} role='option' tabIndex='0' data-index={lang[0]} className={classNames('language-dropdown__dropdown__results__item', { active: lang[0] === value })} aria-selected={lang[0] === value} onClick={this.handleClick} onKeyDown={this.handleKeyDown}>
-        <span className='language-dropdown__dropdown__results__item__native-name'>{lang[2]}</span> <span className='language-dropdown__dropdown__results__item__common-name'>({lang[1]})</span>
+        <span className='language-dropdown__dropdown__results__item__native-name' lang={lang[0]}>{lang[2]}</span> <span className='language-dropdown__dropdown__results__item__common-name'>({lang[1]})</span>
       </div>
     );
   }

app/javascript/mastodon/reducers/compose.js

@@ -186,11 +186,12 @@ const ignoreSuggestion = (state, position, token, completion, path) => {
 };
 
 const sortHashtagsByUse = (state, tags) => {
-  const personalHistory = state.get('tagHistory');
+  const personalHistory = state.get('tagHistory').map(tag => tag.toLowerCase());
 
-  return tags.sort((a, b) => {
-    const usedA = personalHistory.includes(a.name);
-    const usedB = personalHistory.includes(b.name);
+  const tagsWithLowercase = tags.map(t => ({ ...t, lowerName: t.name.toLowerCase() }));
+  const sorted = tagsWithLowercase.sort((a, b) => {
+    const usedA = personalHistory.includes(a.lowerName);
+    const usedB = personalHistory.includes(b.lowerName);
 
     if (usedA === usedB) {
       return 0;
@@ -200,6 +201,8 @@ const sortHashtagsByUse = (state, tags) => {
       return 1;
     }
   });
+  sorted.forEach(tag => delete tag.lowerName);
+  return sorted;
 };
 
 const insertEmoji = (state, position, emojiData, needsSpace) => {

app/javascript/styles/mastodon/admin.scss

@@ -386,7 +386,7 @@ $content-width: 840px;
           position: fixed;
           z-index: 10;
           width: 100%;
-          height: calc(100vh - 56px);
+          height: calc(100% - 56px);
           left: 0;
           bottom: 0;
           overflow-y: auto;

app/javascript/styles/mastodon/components.scss

@@ -4407,6 +4407,7 @@ a.status-card.compact:hover {
   display: flex;
   align-items: center;
   justify-content: center;
+  text-align: center;
   color: $secondary-text-color;
   font-size: 18px;
   font-weight: 500;

app/lib/account_reach_finder.rb

@@ -6,7 +6,7 @@ class AccountReachFinder
   end
 
   def inboxes
-    (followers_inboxes + reporters_inboxes + relay_inboxes).uniq
+    (followers_inboxes + reporters_inboxes + recently_mentioned_inboxes + relay_inboxes).uniq
   end
 
   private
@@ -19,6 +19,13 @@ class AccountReachFinder
     Account.where(id: @account.targeted_reports.select(:account_id)).inboxes
   end
 
+  def recently_mentioned_inboxes
+    cutoff_id       = Mastodon::Snowflake.id_at(2.days.ago, with_random: false)
+    recent_statuses = @account.statuses.recent.where(id: cutoff_id...).limit(200)
+
+    Account.joins(:mentions).where(mentions: { status: recent_statuses }).inboxes.take(2000)
+  end
+
   def relay_inboxes
     Relay.enabled.pluck(:inbox_url)
   end

app/lib/activitypub/activity/flag.rb

@@ -16,7 +16,7 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
         @account,
         target_account,
         status_ids: target_statuses.nil? ? [] : target_statuses.map(&:id),
-        comment: @json['content'] || '',
+        comment: report_comment,
         uri: report_uri
       )
     end
@@ -35,4 +35,8 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
   def report_uri
     @json['id'] unless @json['id'].nil? || invalid_origin?(@json['id'])
   end
+
+  def report_comment
+    (@json['content'] || '')[0...5000]
+  end
 end

app/lib/activitypub/tag_manager.rb

@@ -27,6 +27,8 @@ class ActivityPub::TagManager
     when :note, :comment, :activity
       return activity_account_status_url(target.account, target) if target.reblog?
       short_account_status_url(target.account, target)
+    when :flag
+      target.uri
     end
   end
 
@@ -41,6 +43,8 @@ class ActivityPub::TagManager
       account_status_url(target.account, target)
     when :emoji
       emoji_url(target)
+    when :flag
+      target.uri
     end
   end
 

app/lib/admin/account_statuses_filter.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class Admin::AccountStatusesFilter < AccountStatusesFilter
+  private
+
+  def blocked?
+    false
+  end
+end

app/lib/admin/system_check.rb

@@ -2,6 +2,7 @@
 
 class Admin::SystemCheck
   ACTIVE_CHECKS = [
+    Admin::SystemCheck::MediaPrivacyCheck,
     Admin::SystemCheck::DatabaseSchemaCheck,
     Admin::SystemCheck::SidekiqProcessCheck,
     Admin::SystemCheck::RulesCheck,

app/lib/admin/system_check/elasticsearch_check.rb

@@ -24,7 +24,7 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck
   def running_version
     @running_version ||= begin
       Chewy.client.info['version']['number']
-    rescue Faraday::ConnectionFailed
+    rescue Faraday::ConnectionFailed, Elasticsearch::Transport::Transport::Error
       nil
     end
   end

app/lib/admin/system_check/media_privacy_check.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+class Admin::SystemCheck::MediaPrivacyCheck < Admin::SystemCheck::BaseCheck
+  include RoutingHelper
+
+  def skip?
+    !current_user.can?(:view_devops)
+  end
+
+  def pass?
+    check_media_uploads!
+    @failure_message.nil?
+  end
+
+  def message
+    Admin::SystemCheck::Message.new(@failure_message, @failure_value, @failure_action, true)
+  end
+
+  private
+
+  def check_media_uploads!
+    if Rails.configuration.x.use_s3
+      check_media_listing_inaccessible_s3!
+    else
+      check_media_listing_inaccessible!
+    end
+  end
+
+  def check_media_listing_inaccessible!
+    full_url = full_asset_url(media_attachment.file.url(:original, false))
+
+    # Check if we can list the uploaded file. If true, that's an error
+    directory_url = Addressable::URI.parse(full_url)
+    directory_url.query = nil
+    filename = directory_url.path.gsub(%r{.*/}, '')
+    directory_url.path = directory_url.path.gsub(%r{/[^/]+\Z}, '/')
+    Request.new(:get, directory_url, allow_local: true).perform do |res|
+      if res.truncated_body&.include?(filename)
+        @failure_message = use_storage? ? :upload_check_privacy_error_object_storage : :upload_check_privacy_error
+        @failure_action = 'https://docs.joinmastodon.org/admin/optional/object-storage/#FS'
+      end
+    end
+  rescue
+    nil
+  end
+
+  def check_media_listing_inaccessible_s3!
+    urls_to_check = []
+    paperclip_options = Paperclip::Attachment.default_options
+    s3_protocol = paperclip_options[:s3_protocol]
+    s3_host_alias = paperclip_options[:s3_host_alias]
+    s3_host_name  = paperclip_options[:s3_host_name]
+    bucket_name = paperclip_options.dig(:s3_credentials, :bucket)
+
+    urls_to_check << "#{s3_protocol}://#{s3_host_alias}/" if s3_host_alias.present?
+    urls_to_check << "#{s3_protocol}://#{s3_host_name}/#{bucket_name}/"
+    urls_to_check.uniq.each do |full_url|
+      check_s3_listing!(full_url)
+      break if @failure_message.present?
+    end
+  rescue
+    nil
+  end
+
+  def check_s3_listing!(full_url)
+    bucket_url = Addressable::URI.parse(full_url)
+    bucket_url.path = bucket_url.path.delete_suffix(media_attachment.file.path(:original))
+    bucket_url.query = "max-keys=1&x-random=#{SecureRandom.hex(10)}"
+    Request.new(:get, bucket_url, allow_local: true).perform do |res|
+      if res.truncated_body&.include?('ListBucketResult')
+        @failure_message = :upload_check_privacy_error_object_storage
+        @failure_action  = 'https://docs.joinmastodon.org/admin/optional/object-storage/#S3'
+      end
+    end
+  end
+
+  def media_attachment
+    @media_attachment ||= begin
+      attachment = Account.representative.media_attachments.first
+      if attachment.present?
+        attachment.touch # rubocop:disable Rails/SkipsModelValidations
+        attachment
+      else
+        create_test_attachment!
+      end
+    end
+  end
+
+  def create_test_attachment!
+    Tempfile.create(%w(test-upload .jpg), binmode: true) do |tmp_file|
+      tmp_file.write(
+        Base64.decode64(
+          '/9j/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAYAAAA' \
+          'AAAD/2wCEAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBA' \
+          'QEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE' \
+          'BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAf/AABEIAAEAAgMBEQACEQEDEQH/x' \
+          'ABKAAEAAAAAAAAAAAAAAAAAAAALEAEAAAAAAAAAAAAAAAAAAAAAAQEAAAAAAAAAAAAAAAA' \
+          'AAAAAEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwA/8H//2Q=='
+        )
+      )
+      tmp_file.flush
+      Account.representative.media_attachments.create!(file: tmp_file)
+    end
+  end
+end

app/lib/admin/system_check/message.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
 
 class Admin::SystemCheck::Message
-  attr_reader :key, :value, :action
+  attr_reader :key, :value, :action, :critical
 
-  def initialize(key, value = nil, action = nil)
-    @key    = key
-    @value  = value
-    @action = action
+  def initialize(key, value = nil, action = nil, critical = false)
+    @key      = key
+    @value    = value
+    @action   = action
+    @critical = critical
   end
 end

app/lib/application_extension.rb

@@ -9,10 +9,6 @@ module ApplicationExtension
     validates :redirect_uri, length: { maximum: 2_000 }
   end
 
-  def most_recently_used_access_token
-    @most_recently_used_access_token ||= access_tokens.where.not(last_used_at: nil).order(last_used_at: :desc).first
-  end
-
   def confirmation_redirect_uri
     redirect_uri.lines.first.strip
   end

app/lib/link_details_extractor.rb

@@ -140,7 +140,7 @@ class LinkDetailsExtractor
   end
 
   def html
-    player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
+    player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowfullscreen: 'true', allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
   end
 
   def width

app/lib/plain_text_formatter.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
 class PlainTextFormatter
-  include ActionView::Helpers::TextHelper
-
-  NEWLINE_TAGS_RE = /(<br \/>|<br>|<\/p>)+/.freeze
+  NEWLINE_TAGS_RE = %r{(<br />|<br>|</p>)+}
 
   attr_reader :text, :local
 
@@ -18,7 +16,10 @@ class PlainTextFormatter
     if local?
       text
     else
-      strip_tags(insert_newlines).chomp
+      node = Nokogiri::HTML.fragment(insert_newlines)
+      # Elements that are entirely removed with our Sanitize config
+      node.xpath('.//iframe|.//math|.//noembed|.//noframes|.//noscript|.//plaintext|.//script|.//style|.//svg|.//xmp').remove
+      node.text.chomp
     end
   end
 

app/lib/request.rb

@@ -4,14 +4,60 @@ require 'ipaddr'
 require 'socket'
 require 'resolv'
 
-# Monkey-patch the HTTP.rb timeout class to avoid using a timeout block
+# Use our own timeout class to avoid using HTTP.rb's timeout block
 # around the Socket#open method, since we use our own timeout blocks inside
 # that method
-class HTTP::Timeout::PerOperation
+#
+# Also changes how the read timeout behaves so that it is cumulative (closer
+# to HTTP::Timeout::Global, but still having distinct timeouts for other
+# operation types)
+class PerOperationWithDeadline < HTTP::Timeout::PerOperation
+  READ_DEADLINE = 30
+
+  def initialize(*args)
+    super
+
+    @read_deadline = options.fetch(:read_deadline, READ_DEADLINE)
+  end
+
   def connect(socket_class, host, port, nodelay = false)
     @socket = socket_class.open(host, port)
     @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay
   end
+
+  # Reset deadline when the connection is re-used for different requests
+  def reset_counter
+    @deadline = nil
+  end
+
+  # Read data from the socket
+  def readpartial(size, buffer = nil)
+    @deadline ||= Process.clock_gettime(Process::CLOCK_MONOTONIC) + @read_deadline
+
+    timeout = false
+    loop do
+      result = @socket.read_nonblock(size, buffer, exception: false)
+
+      return :eof if result.nil?
+
+      remaining_time = @deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      raise HTTP::TimeoutError, "Read timed out after #{@read_timeout} seconds" if timeout
+      raise HTTP::TimeoutError, "Read timed out after a total of #{@read_deadline} seconds" if remaining_time <= 0
+      return result if result != :wait_readable
+
+      # marking the socket for timeout. Why is this not being raised immediately?
+      # it seems there is some race-condition on the network level between calling
+      # #read_nonblock and #wait_readable, in which #read_nonblock signalizes waiting
+      # for reads, and when waiting for x seconds, it returns nil suddenly without completing
+      # the x seconds. In a normal case this would be a timeout on wait/read, but it can
+      # also mean that the socket has been closed by the server. Therefore we "mark" the
+      # socket for timeout and try to read more bytes. If it returns :eof, it's all good, no
+      # timeout. Else, the first timeout was a proper timeout.
+      # This hack has to be done because io/wait#wait_readable doesn't provide a value for when
+      # the socket is closed by the server, and HTTP::Parser doesn't provide the limit for the chunks.
+      timeout = true unless @socket.to_io.wait_readable([remaining_time, @read_timeout].min)
+    end
+  end
 end
 
 class Request
@@ -20,7 +66,7 @@ class Request
   # We enforce a 5s timeout on DNS resolving, 5s timeout on socket opening
   # and 5s timeout on the TLS handshake, meaning the worst case should take
   # about 15s in total
-  TIMEOUT = { connect: 5, read: 10, write: 10 }.freeze
+  TIMEOUT = { connect_timeout: 5, read_timeout: 10, write_timeout: 10, read_deadline: 30 }.freeze
 
   include RoutingHelper
 
@@ -31,6 +77,7 @@ class Request
     @url         = Addressable::URI.parse(url).normalize
     @http_client = options.delete(:http_client)
     @options     = options.merge(socket_class: use_proxy? ? ProxySocket : Socket)
+    @options     = @options.merge(timeout_class: PerOperationWithDeadline, timeout_options: TIMEOUT)
     @options     = @options.merge(proxy_url) if use_proxy?
     @headers     = {}
 
@@ -91,7 +138,7 @@ class Request
     end
 
     def http_client
-      HTTP.use(:auto_inflate).timeout(TIMEOUT.dup).follow(max_hops: 3)
+      HTTP.use(:auto_inflate).follow(max_hops: 3)
     end
   end
 
@@ -231,11 +278,11 @@ class Request
         end
 
         until socks.empty?
-          _, available_socks, = IO.select(nil, socks, nil, Request::TIMEOUT[:connect])
+          _, available_socks, = IO.select(nil, socks, nil, Request::TIMEOUT[:connect_timeout])
 
           if available_socks.nil?
             socks.each(&:close)
-            raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds"
+            raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect_timeout]} seconds"
           end
 
           available_socks.each do |sock|

app/lib/tag_manager.rb

@@ -7,18 +7,18 @@ class TagManager
   include RoutingHelper
 
   def web_domain?(domain)
-    domain.nil? || domain.gsub(/[\/]/, '').casecmp(Rails.configuration.x.web_domain).zero?
+    domain.nil? || domain.delete_suffix('/').casecmp(Rails.configuration.x.web_domain).zero?
   end
 
   def local_domain?(domain)
-    domain.nil? || domain.gsub(/[\/]/, '').casecmp(Rails.configuration.x.local_domain).zero?
+    domain.nil? || domain.delete_suffix('/').casecmp(Rails.configuration.x.local_domain).zero?
   end
 
   def normalize_domain(domain)
     return if domain.nil?
 
     uri = Addressable::URI.new
-    uri.host = domain.gsub(/[\/]/, '')
+    uri.host = domain.delete_suffix('/')
     uri.normalized_host
   end
 
@@ -28,7 +28,7 @@ class TagManager
     domain = uri.host + (uri.port ? ":#{uri.port}" : '')
 
     TagManager.instance.web_domain?(domain)
-  rescue Addressable::URI::InvalidURIError
+  rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
     false
   end
 end

app/lib/text_formatter.rb

@@ -48,6 +48,26 @@ class TextFormatter
     html.html_safe # rubocop:disable Rails/OutputSafety
   end
 
+  class << self
+    include ERB::Util
+
+    def shortened_link(url, rel_me: false)
+      url = Addressable::URI.parse(url).to_s
+      rel = rel_me ? (DEFAULT_REL + %w(me)) : DEFAULT_REL
+
+      prefix      = url.match(URL_PREFIX_REGEX).to_s
+      display_url = url[prefix.length, 30]
+      suffix      = url[prefix.length + 30..-1]
+      cutoff      = url[prefix.length..-1].length > 30
+
+      <<~HTML.squish.html_safe # rubocop:disable Rails/OutputSafety
+        <a href="#{h(url)}" target="_blank" rel="#{rel.join(' ')}"><span class="invisible">#{h(prefix)}</span><span class="#{cutoff ? 'ellipsis' : ''}">#{h(display_url)}</span><span class="invisible">#{h(suffix)}</span></a>
+      HTML
+    rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
+      h(url)
+    end
+  end
+
   private
 
   def rewrite
@@ -70,19 +90,7 @@ class TextFormatter
   end
 
   def link_to_url(entity)
-    url = Addressable::URI.parse(entity[:url]).to_s
-    rel = with_rel_me? ? (DEFAULT_REL + %w(me)) : DEFAULT_REL
-
-    prefix      = url.match(URL_PREFIX_REGEX).to_s
-    display_url = url[prefix.length, 30]
-    suffix      = url[prefix.length + 30..-1]
-    cutoff      = url[prefix.length..-1].length > 30
-
-    <<~HTML.squish
-      <a href="#{h(url)}" target="_blank" rel="#{rel.join(' ')}"><span class="invisible">#{h(prefix)}</span><span class="#{cutoff ? 'ellipsis' : ''}">#{h(display_url)}</span><span class="invisible">#{h(suffix)}</span></a>
-    HTML
-  rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
-    h(entity[:url])
+    TextFormatter.shortened_link(entity[:url], rel_me: with_rel_me?)
   end
 
   def link_to_hashtag(entity)

app/lib/translation_service/deepl.rb

@@ -46,7 +46,7 @@ class TranslationService::DeepL < TranslationService
 
     raise UnexpectedResponseError unless json.is_a?(Hash)
 
-    Translation.new(text: json.dig('translations', 0, 'text'), detected_source_language: json.dig('translations', 0, 'detected_source_language')&.downcase, provider: 'DeepL.com')
+    Translation.new(text: Sanitize.fragment(json.dig('translations', 0, 'text'), Sanitize::Config::MASTODON_STRICT), detected_source_language: json.dig('translations', 0, 'detected_source_language')&.downcase, provider: 'DeepL.com')
   rescue Oj::ParseError
     raise UnexpectedResponseError
   end

app/lib/translation_service/libre_translate.rb

@@ -37,7 +37,7 @@ class TranslationService::LibreTranslate < TranslationService
 
     raise UnexpectedResponseError unless json.is_a?(Hash)
 
-    Translation.new(text: json['translatedText'], detected_source_language: source_language, provider: 'LibreTranslate')
+    Translation.new(text: Sanitize.fragment(json['translatedText'], Sanitize::Config::MASTODON_STRICT), detected_source_language: source_language, provider: 'LibreTranslate')
   rescue Oj::ParseError
     raise UnexpectedResponseError
   end

app/lib/video_metadata_extractor.rb

@@ -43,6 +43,9 @@ class VideoMetadataExtractor
         @height      = video_stream[:height]
         @frame_rate  = video_stream[:avg_frame_rate] == '0/0' ? nil : Rational(video_stream[:avg_frame_rate])
         @r_frame_rate = video_stream[:r_frame_rate] == '0/0' ? nil : Rational(video_stream[:r_frame_rate])
+        # For some video streams the frame_rate reported by `ffprobe` will be 0/0, but for these streams we
+        # should use `r_frame_rate` instead. Video screencast generated by Gnome Screencast have this issue.
+        @frame_rate ||= @r_frame_rate
       end
 
       if (audio_stream = audio_streams.first)

app/mailers/application_mailer.rb

@@ -7,6 +7,8 @@ class ApplicationMailer < ActionMailer::Base
   helper :instance
   helper :formatting
 
+  after_action :set_autoreply_headers!
+
   protected
 
   def locale_for_account(account)
@@ -14,4 +16,10 @@ class ApplicationMailer < ActionMailer::Base
       yield
     end
   end
+
+  def set_autoreply_headers!
+    headers['Precedence'] = 'list'
+    headers['X-Auto-Response-Suppress'] = 'All'
+    headers['Auto-Submitted'] = 'auto-generated'
+  end
 end

app/models/account.rb

@@ -107,7 +107,7 @@ class Account < ApplicationRecord
   scope :bots, -> { where(actor_type: %w(Application Service)) }
   scope :groups, -> { where(actor_type: 'Group') }
   scope :alphabetic, -> { order(domain: :asc, username: :asc) }
-  scope :matches_username, ->(value) { where(arel_table[:username].matches("#{value}%")) }
+  scope :matches_username, ->(value) { where('lower((username)::text) LIKE lower(?)', "#{value}%") }
   scope :matches_display_name, ->(value) { where(arel_table[:display_name].matches("#{value}%")) }
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
   scope :without_unapproved, -> { left_outer_joins(:user).remote.or(left_outer_joins(:user).merge(User.approved.confirmed)) }

app/models/account_conversation.rb

@@ -16,34 +16,44 @@
 class AccountConversation < ApplicationRecord
   include Redisable
 
+  attr_writer :participant_accounts
+
+  before_validation :set_last_status
   after_commit :push_to_streaming_api
 
   belongs_to :account
   belongs_to :conversation
   belongs_to :last_status, class_name: 'Status'
 
-  before_validation :set_last_status
-
   def participant_account_ids=(arr)
     self[:participant_account_ids] = arr.sort
+    @participant_accounts = nil
   end
 
   def participant_accounts
-    if participant_account_ids.empty?
-      [account]
-    else
-      participants = Account.where(id: participant_account_ids)
-      participants.empty? ? [account] : participants
-    end
+    @participant_accounts ||= Account.where(id: participant_account_ids).to_a
+    @participant_accounts.presence || [account]
   end
 
   class << self
     def to_a_paginated_by_id(limit, options = {})
-      if options[:min_id]
-        paginate_by_min_id(limit, options[:min_id], options[:max_id]).reverse
-      else
-        paginate_by_max_id(limit, options[:max_id], options[:since_id]).to_a
+      array = begin
+        if options[:min_id]
+          paginate_by_min_id(limit, options[:min_id], options[:max_id]).reverse
+        else
+          paginate_by_max_id(limit, options[:max_id], options[:since_id]).to_a
+        end
       end
+
+      # Preload participants
+      participant_ids = array.flat_map(&:participant_account_ids)
+      accounts_by_id = Account.where(id: participant_ids).index_by(&:id)
+
+      array.each do |conversation|
+        conversation.participant_accounts = conversation.participant_account_ids.filter_map { |id| accounts_by_id[id] }
+      end
+
+      array
     end
 
     def paginate_by_min_id(limit, min_id = nil, max_id = nil)

app/models/admin/status_batch_action.rb

@@ -137,6 +137,6 @@ class Admin::StatusBatchAction
   end
 
   def allowed_status_ids
-    AccountStatusesFilter.new(@report.target_account, current_account).results.with_discarded.where(id: status_ids).pluck(:id)
+    Admin::AccountStatusesFilter.new(@report.target_account, current_account).results.with_discarded.where(id: status_ids).pluck(:id)
   end
 end

app/models/backup.rb

@@ -17,6 +17,6 @@
 class Backup < ApplicationRecord
   belongs_to :user, inverse_of: :backups
 
-  has_attached_file :dump
+  has_attached_file :dump, s3_permissions: ->(*) { ENV['S3_PERMISSION'] == '' ? nil : 'private' }
   do_not_validate_attachment_file_type :dump
 end

app/models/concerns/attachmentable.rb

@@ -22,15 +22,14 @@ module Attachmentable
 
   included do
     def self.has_attached_file(name, options = {}) # rubocop:disable Naming/PredicateName
-      options = { validate_media_type: false }.merge(options)
       super(name, options)
-      send(:"before_#{name}_post_process") do
+
+      send(:"before_#{name}_validate", prepend: true) do
         attachment = send(name)
         check_image_dimension(attachment)
         set_file_content_type(attachment)
         obfuscate_file_name(attachment)
         set_file_extension(attachment)
-        Paperclip::Validators::MediaTypeSpoofDetectionValidator.new(attributes: [name]).validate(self)
       end
     end
   end

app/models/concerns/ldap_authenticable.rb

@@ -6,7 +6,7 @@ module LdapAuthenticable
   class_methods do
     def authenticate_with_ldap(params = {})
       ldap   = Net::LDAP.new(ldap_options)
-      filter = format(Devise.ldap_search_filter, uid: Devise.ldap_uid, mail: Devise.ldap_mail, email: params[:email])
+      filter = format(Devise.ldap_search_filter, uid: Devise.ldap_uid, mail: Devise.ldap_mail, email: Net::LDAP::Filter.escape(params[:email]))
 
       if (user_info = ldap.bind_as(base: Devise.ldap_base, filter: filter, password: params[:password]))
         ldap_get_user(user_info.first)

app/models/form/account_batch.rb

@@ -17,8 +17,8 @@ class Form::AccountBatch
       unfollow!
     when 'remove_from_followers'
       remove_from_followers!
-    when 'block_domains'
-      block_domains!
+    when 'remove_domains_from_followers'
+      remove_domains_from_followers!
     when 'approve'
       approve!
     when 'reject'
@@ -35,9 +35,15 @@ class Form::AccountBatch
   private
 
   def follow!
+    error = nil
+
     accounts.each do |target_account|
       FollowService.new.call(current_account, target_account)
+    rescue Mastodon::NotPermittedError, ActiveRecord::RecordNotFound => e
+      error ||= e
     end
+
+    raise error if error.present?
   end
 
   def unfollow!
@@ -50,10 +56,8 @@ class Form::AccountBatch
     RemoveFromFollowersService.new.call(current_account, account_ids)
   end
 
-  def block_domains!
-    AfterAccountDomainBlockWorker.push_bulk(account_domains) do |domain|
-      [current_account.id, domain]
-    end
+  def remove_domains_from_followers!
+    RemoveDomainsFromFollowersService.new.call(current_account, account_domains)
   end
 
   def account_domains

app/models/identity.rb

@@ -12,7 +12,7 @@
 #
 
 class Identity < ApplicationRecord
-  belongs_to :user, dependent: :destroy
+  belongs_to :user
   validates :uid, presence: true, uniqueness: { scope: :provider }
   validates :provider, presence: true
 

app/models/report.rb

@@ -39,7 +39,10 @@ class Report < ApplicationRecord
   scope :resolved,   -> { where.not(action_taken_at: nil) }
   scope :with_accounts, -> { includes([:account, :target_account, :action_taken_by_account, :assigned_account].index_with({ user: [:invite_request, :invite] })) }
 
-  validates :comment, length: { maximum: 1_000 }
+  # A report is considered local if the reporter is local
+  delegate :local?, to: :account
+
+  validates :comment, length: { maximum: 1_000 }, if: :local?
   validates :rule_ids, absence: true, unless: :violation?
 
   validate :validate_rule_ids
@@ -50,10 +53,6 @@ class Report < ApplicationRecord
     violation: 2_000,
   }
 
-  def local?
-    false # Force uri_for to use uri attribute
-  end
-
   before_validation :set_uri, only: :create
 
   after_create_commit :trigger_webhooks

app/models/status.rb

@@ -353,13 +353,25 @@ class Status < ApplicationRecord
 
       account_ids.uniq!
 
+      status_ids = cached_items.map { |item| item.reblog? ? item.reblog_of_id : item.id }.uniq
+
       return if account_ids.empty?
 
       accounts = Account.where(id: account_ids).includes(:account_stat, :user).index_by(&:id)
 
+      status_stats = StatusStat.where(status_id: status_ids).index_by(&:status_id)
+
       cached_items.each do |item|
         item.account = accounts[item.account_id]
         item.reblog.account = accounts[item.reblog.account_id] if item.reblog?
+
+        if item.reblog?
+          status_stat = status_stats[item.reblog.id]
+          item.reblog.status_stat = status_stat if status_stat.present?
+        else
+          status_stat = status_stats[item.id]
+          item.status_stat = status_stat if status_stat.present?
+        end
       end
     end
 

app/models/user.rb

@@ -480,10 +480,13 @@ class User < ApplicationRecord
   def prepare_new_user!
     BootstrapTimelineWorker.perform_async(account_id)
     ActivityTracker.increment('activity:accounts:local')
+    ActivityTracker.record('activity:logins', id)
     UserMailer.welcome(self).deliver_later
   end
 
   def prepare_returning_user!
+    return unless confirmed?
+
     ActivityTracker.record('activity:logins', id)
     regenerate_feed! if needs_feed_update?
   end

app/models/webhook.rb

@@ -19,6 +19,8 @@ class Webhook < ApplicationRecord
     report.created
   ).freeze
 
+  attr_writer :current_account
+
   scope :enabled, -> { where(enabled: true) }
 
   validates :url, presence: true, url: true
@@ -26,6 +28,7 @@ class Webhook < ApplicationRecord
   validates :events, presence: true
 
   validate :validate_events
+  validate :validate_permissions
 
   before_validation :strip_events
   before_validation :generate_secret
@@ -42,12 +45,29 @@ class Webhook < ApplicationRecord
     update!(enabled: false)
   end
 
+  def required_permissions
+    events.map { |event| Webhook.permission_for_event(event) }
+  end
+
+  def self.permission_for_event(event)
+    case event
+    when 'account.approved', 'account.created', 'account.updated'
+      :manage_users
+    when 'report.created'
+      :manage_reports
+    end
+  end
+
   private
 
   def validate_events
     errors.add(:events, :invalid) if events.any? { |e| !EVENTS.include?(e) }
   end
 
+  def validate_permissions
+    errors.add(:events, :invalid_permissions) if defined?(@current_account) && required_permissions.any? { |permission| !@current_account.user_role.can?(permission) }
+  end
+
   def strip_events
     self.events = events.map { |str| str.strip.presence }.compact if events.present?
   end

app/policies/admin/status_policy.rb

@@ -12,7 +12,7 @@ class Admin::StatusPolicy < ApplicationPolicy
   end
 
   def show?
-    role.can?(:manage_reports, :manage_users) && (record.public_visibility? || record.unlisted_visibility? || record.reported?)
+    role.can?(:manage_reports, :manage_users) && (record.public_visibility? || record.unlisted_visibility? || record.reported? || viewable_through_normal_policy?)
   end
 
   def destroy?
@@ -26,4 +26,10 @@ class Admin::StatusPolicy < ApplicationPolicy
   def review?
     role.can?(:manage_taxonomies)
   end
+
+  private
+
+  def viewable_through_normal_policy?
+    StatusPolicy.new(current_account, record, @preloaded_relations).show?
+  end
 end

app/policies/webhook_policy.rb

@@ -14,7 +14,7 @@ class WebhookPolicy < ApplicationPolicy
   end
 
   def update?
-    role.can?(:manage_webhooks)
+    role.can?(:manage_webhooks) && record.required_permissions.all? { |permission| role.can?(permission) }
   end
 
   def enable?
@@ -30,6 +30,6 @@ class WebhookPolicy < ApplicationPolicy
   end
 
   def destroy?
-    role.can?(:manage_webhooks)
+    role.can?(:manage_webhooks) && record.required_permissions.all? { |permission| role.can?(permission) }
   end
 end

app/serializers/rest/preview_card_serializer.rb

@@ -11,4 +11,8 @@ class REST::PreviewCardSerializer < ActiveModel::Serializer
   def image
     object.image? ? full_asset_url(object.image.url(:original)) : nil
   end
+
+  def html
+    Sanitize.fragment(object.html, Sanitize::Config::MASTODON_OEMBED)
+  end
 end

app/services/activitypub/process_account_service.rb

@@ -59,6 +59,9 @@ class ActivityPub::ProcessAccountService < BaseService
     @account.suspended_at      = domain_block.created_at if auto_suspend?
     @account.suspension_origin = :local if auto_suspend?
     @account.silenced_at       = domain_block.created_at if auto_silence?
+
+    set_immediate_protocol_attributes!
+
     @account.save
   end
 

app/services/follow_migration_service.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class FollowMigrationService < FollowService
+  # Follow an account with the same settings as another account, and unfollow the old account once the request is sent
+  # @param [Account] source_account From which to follow
+  # @param [Account] target_account Account to follow
+  # @param [Account] old_target_account Account to unfollow once the follow request has been sent to the new one
+  # @option [Boolean] bypass_locked Whether to immediately follow the new account even if it is locked
+  def call(source_account, target_account, old_target_account, bypass_locked: false)
+    @old_target_account = old_target_account
+
+    follow    = source_account.active_relationships.find_by(target_account: old_target_account)
+    reblogs   = follow&.show_reblogs?
+    notify    = follow&.notify?
+    languages = follow&.languages
+
+    super(source_account, target_account, reblogs: reblogs, notify: notify, languages: languages, bypass_locked: bypass_locked, bypass_limit: true)
+  end
+
+  private
+
+  def request_follow!
+    follow_request = @source_account.request_follow!(@target_account, **follow_options.merge(rate_limit: @options[:with_rate_limit], bypass_limit: @options[:bypass_limit]))
+
+    if @target_account.local?
+      LocalNotificationWorker.perform_async(@target_account.id, follow_request.id, follow_request.class.name, 'follow_request')
+      UnfollowService.new.call(@source_account, @old_target_account, skip_unmerge: true)
+    elsif @target_account.activitypub?
+      ActivityPub::MigratedFollowDeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url, @old_target_account.id)
+    end
+
+    follow_request
+  end
+
+  def direct_follow!
+    follow = super
+    UnfollowService.new.call(@source_account, @old_target_account, skip_unmerge: true)
+    follow
+  end
+end

app/services/remove_domains_from_followers_service.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class RemoveDomainsFromFollowersService < BaseService
+  include Payloadable
+
+  def call(source_account, target_domains)
+    source_account.passive_relationships.where(account_id: Account.where(domain: target_domains)).find_each do |follow|
+      follow.destroy
+
+      create_notification(follow) if source_account.local? && !follow.account.local? && follow.account.activitypub?
+    end
+  end
+
+  private
+
+  def create_notification(follow)
+    ActivityPub::DeliveryWorker.perform_async(build_json(follow), follow.target_account_id, follow.account.inbox_url)
+  end
+
+  def build_json(follow)
+    Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer))
+  end
+end

app/services/remove_status_service.rb

@@ -12,6 +12,7 @@ class RemoveStatusService < BaseService
   # @option  [Boolean] :immediate
   # @option  [Boolean] :preserve
   # @option  [Boolean] :original_removed
+  # @option  [Boolean] :skip_streaming
   def call(status, **options)
     @payload  = Oj.dump(event: :delete, payload: status.id.to_s)
     @status   = status
@@ -52,6 +53,9 @@ class RemoveStatusService < BaseService
 
   private
 
+  # The following FeedManager calls all do not result in redis publishes for
+  # streaming, as the `:update` option is false
+
   def remove_from_self
     FeedManager.instance.unpush_from_home(@account, @status)
   end
@@ -75,6 +79,8 @@ class RemoveStatusService < BaseService
     # followers. Here we send a delete to actively mentioned accounts
     # that may not follow the account
 
+    return if skip_streaming?
+
     @status.active_mentions.find_each do |mention|
       redis.publish("timeline:#{mention.account_id}", @payload)
     end
@@ -103,7 +109,7 @@ class RemoveStatusService < BaseService
     # without us being able to do all the fancy stuff
 
     @status.reblogs.rewhere(deleted_at: [nil, @status.deleted_at]).includes(:account).reorder(nil).find_each do |reblog|
-      RemoveStatusService.new.call(reblog, original_removed: true)
+      RemoveStatusService.new.call(reblog, original_removed: true, skip_streaming: skip_streaming?)
     end
   end
 
@@ -114,6 +120,8 @@ class RemoveStatusService < BaseService
 
     return unless @status.public_visibility?
 
+    return if skip_streaming?
+
     @status.tags.map(&:name).each do |hashtag|
       redis.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}", @payload)
       redis.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}:local", @payload) if @status.local?
@@ -123,6 +131,8 @@ class RemoveStatusService < BaseService
   def remove_from_public
     return unless @status.public_visibility?
 
+    return if skip_streaming?
+
     redis.publish('timeline:public', @payload)
     redis.publish(@status.local? ? 'timeline:public:local' : 'timeline:public:remote', @payload)
   end
@@ -130,6 +140,8 @@ class RemoveStatusService < BaseService
   def remove_from_media
     return unless @status.public_visibility?
 
+    return if skip_streaming?
+
     redis.publish('timeline:public:media', @payload)
     redis.publish(@status.local? ? 'timeline:public:local:media' : 'timeline:public:remote:media', @payload)
   end
@@ -143,4 +155,8 @@ class RemoveStatusService < BaseService
   def permanently?
     @options[:immediate] || !(@options[:preserve] || @status.reported?)
   end
+
+  def skip_streaming?
+    !!@options[:skip_streaming]
+  end
 end

app/services/resolve_url_service.rb

@@ -89,13 +89,28 @@ class ResolveURLService < BaseService
   def process_local_url
     recognized_params = Rails.application.routes.recognize_path(@url)
 
-    return unless recognized_params[:action] == 'show'
+    case recognized_params[:controller]
+    when 'statuses'
+      return unless recognized_params[:action] == 'show'
 
-    if recognized_params[:controller] == 'statuses'
       status = Status.find_by(id: recognized_params[:id])
       check_local_status(status)
-    elsif recognized_params[:controller] == 'accounts'
+    when 'accounts'
+      return unless recognized_params[:action] == 'show'
+
       Account.find_local(recognized_params[:username])
+    when 'home'
+      return unless recognized_params[:action] == 'index' && recognized_params[:username_with_domain].present?
+
+      if recognized_params[:any]&.match?(/\A[0-9]+\Z/)
+        status = Status.find_by(id: recognized_params[:any])
+        check_local_status(status)
+      elsif recognized_params[:any].blank?
+        username, domain = recognized_params[:username_with_domain].gsub(/\A@/, '').split('@')
+        return unless username.present? && domain.present?
+
+        Account.find_remote(username, domain)
+      end
     end
   end
 

app/services/translate_status_service.rb

@@ -12,7 +12,9 @@ class TranslateStatusService < BaseService
     @content = status_content_format(@status)
     @target_language = target_language
 
-    Rails.cache.fetch("translations/#{@status.language}/#{@target_language}/#{content_hash}", expires_in: CACHE_TTL) { translation_backend.translate(@content, @status.language, @target_language) }
+    Rails.cache.fetch("translations:v2/#{@status.language}/#{@target_language}/#{content_hash}", expires_in: CACHE_TTL) do
+      translation_backend.translate(@content, @status.language, @target_language)
+    end
   end
 
   private

app/validators/vote_validator.rb

@@ -3,8 +3,8 @@
 class VoteValidator < ActiveModel::Validator
   def validate(vote)
     vote.errors.add(:base, I18n.t('polls.errors.expired')) if vote.poll.expired?
-
     vote.errors.add(:base, I18n.t('polls.errors.invalid_choice')) if invalid_choice?(vote)
+    vote.errors.add(:base, I18n.t('polls.errors.self_vote')) if self_vote?(vote)
 
     if vote.poll.multiple? && vote.poll.votes.where(account: vote.account, choice: vote.choice).exists?
       vote.errors.add(:base, I18n.t('polls.errors.already_voted'))
@@ -18,4 +18,8 @@ class VoteValidator < ActiveModel::Validator
   def invalid_choice?(vote)
     vote.choice.negative? || vote.choice >= vote.poll.options.size
   end
+
+  def self_vote?(vote)
+    vote.account_id == vote.poll.account_id
+  end
 end

app/views/admin/dashboard/index.html.haml

@@ -12,7 +12,7 @@
 - unless @system_checks.empty?
   .flash-message-stack
     - @system_checks.each do |message|
-      .flash-message.warning
+      .flash-message{ class: message.critical ? 'alert' : 'warning' }
         = t("admin.system_checks.#{message.key}.message_html", value: message.value ? content_tag(:strong, message.value) : nil)
         - if message.action
           = link_to t("admin.system_checks.#{message.key}.action"), message.action

app/views/admin/statuses/show.html.haml

@@ -34,7 +34,7 @@
           %td
             - if @status.trend.allowed?
               %abbr{ title: t('admin.trends.tags.current_score', score: @status.trend.score) }= t('admin.trends.tags.trending_rank', rank: @status.trend.rank)
-            - elsif @status.trend.requires_review?
+            - elsif @status.requires_review?
               = t('admin.trends.pending_review')
             - else
               = t('admin.trends.not_allowed_to_trend')

app/views/admin/trends/links/preview_card_providers/index.html.haml

@@ -29,7 +29,7 @@
   - Trends::PreviewCardProviderFilter::KEYS.each do |key|
     = hidden_field_tag key, params[key] if params[key].present?
 
-  .batch-table.optional
+  .batch-table
     .batch-table__toolbar
       %label.batch-table__toolbar__select.batch-checkbox-all
         = check_box_tag :batch_checkbox_all, nil, false

app/views/admin/webhooks/_form.html.haml

@@ -5,7 +5,7 @@
     = f.input :url, wrapper: :with_block_label, input_html: { placeholder: 'https://' }
 
   .fields-group
-    = f.input :events, collection: Webhook::EVENTS, wrapper: :with_block_label, include_blank: false, as: :check_boxes, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li'
+    = f.input :events, collection: Webhook::EVENTS, wrapper: :with_block_label, include_blank: false, as: :check_boxes, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li', disabled: Webhook::EVENTS.filter { |event| !current_user.role.can?(Webhook.permission_for_event(event)) }
 
   .actions
     = f.button :button, @webhook.new_record? ? t('admin.webhooks.add_new') : t('generic.save_changes'), type: :submit

app/views/application/_sidebar.html.haml

@@ -3,7 +3,7 @@
     = image_tag @instance_presenter.thumbnail&.file&.url(:'@1x') || asset_pack_path('media/images/preview.png'), alt: @instance_presenter.title
 
   .hero-widget__text
-    %p= @instance_presenter.description.html_safe.presence || t('about.about_mastodon_html')
+    %p= @instance_presenter.description.presence || t('about.about_mastodon_html')
 
 - if Setting.trends && !(user_signed_in? && !current_user.setting_trends)
   - trends = Trends.tags.query.allowed.limit(3)

app/views/disputes/strikes/show.html.haml

@@ -50,15 +50,15 @@
             .strike-card__statuses-list__item
               - if (status = status_map[status_id.to_i])
                 .one-liner
-                  = link_to short_account_status_url(@strike.target_account, status_id), class: 'emojify' do
-                    = one_line_preview(status)
+                  .emojify= one_line_preview(status)
 
-                    - status.ordered_media_attachments.each do |media_attachment|
-                      %abbr{ title: media_attachment.description }
-                        = fa_icon 'link'
-                        = media_attachment.file_file_name
+                  - status.ordered_media_attachments.each do |media_attachment|
+                    %abbr{ title: media_attachment.description }
+                      = fa_icon 'link'
+                      = media_attachment.file_file_name
                 .strike-card__statuses-list__item__meta
-                  %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
+                  = link_to ActivityPub::TagManager.instance.url_for(status), target: '_blank' do
+                    %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
                   - unless status.application.nil?
                     ·
                     = status.application.name

app/views/oauth/authorized_applications/index.html.haml

@@ -18,8 +18,8 @@
 
       .announcements-list__item__action-bar
         .announcements-list__item__meta
-          - if application.most_recently_used_access_token
-            = t('doorkeeper.authorized_applications.index.last_used_at', date: l(application.most_recently_used_access_token.last_used_at.to_date))
+          - if @last_used_at_by_app[application.id]
+            = t('doorkeeper.authorized_applications.index.last_used_at', date: l(@last_used_at_by_app[application.id].to_date))
           - else
             = t('doorkeeper.authorized_applications.index.never_used')
 

app/views/relationships/show.html.haml

@@ -48,7 +48,7 @@
 
         = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_followers')]), name: :remove_from_followers, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } unless following_relationship?
 
-        = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_domains')]), name: :block_domains, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } if followed_by_relationship?
+        = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_domains')]), name: :remove_domains_from_followers, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } if followed_by_relationship?
     .batch-table__body
       - if @accounts.empty?
         = nothing_here 'nothing-here--under-tabs'

app/views/settings/exports/show.html.haml

@@ -64,6 +64,6 @@
             %td= l backup.created_at
             - if backup.processed?
               %td= number_to_human_size backup.dump_file_size
-              %td= table_link_to 'download', t('exports.archive_takeout.download'), backup.dump.url
+              %td= table_link_to 'download', t('exports.archive_takeout.download'), download_backup_url(backup)
             - else
               %td{ colspan: 2 }= t('exports.archive_takeout.in_progress')

app/views/shared/_og.html.haml

@@ -1,5 +1,5 @@
 - thumbnail     = @instance_presenter.thumbnail
-- description ||= strip_tags(@instance_presenter.description.presence || t('about.about_mastodon_html'))
+- description ||= @instance_presenter.description.presence || strip_tags(t('about.about_mastodon_html'))
 
 %meta{ name: 'description', content: description }/
 

app/views/user_mailer/backup_ready.html.haml

@@ -55,5 +55,5 @@
                             %tbody
                               %tr
                                 %td.button-primary
-                                  = link_to full_asset_url(@backup.dump.url) do
+                                  = link_to download_backup_url(@backup) do
                                     %span= t 'exports.archive_takeout.download'

app/views/user_mailer/backup_ready.text.erb

@@ -4,4 +4,4 @@
 
 <%= t 'user_mailer.backup_ready.explanation' %>
 
-=> <%= full_asset_url(@backup.dump.url) %>
+=> <%= download_backup_url(@backup) %>

app/workers/activitypub/migrated_follow_delivery_worker.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class ActivityPub::MigratedFollowDeliveryWorker < ActivityPub::DeliveryWorker
+  def perform(json, source_account_id, inbox_url, old_target_account_id, options = {})
+    super(json, source_account_id, inbox_url, options)
+    unfollow_old_account!(old_target_account_id)
+  end
+
+  private
+
+  def unfollow_old_account!(old_target_account_id)
+    old_target_account = Account.find(old_target_account_id)
+    UnfollowService.new.call(@source_account, old_target_account, skip_unmerge: true)
+  rescue StandardError
+    true
+  end
+end

app/workers/scheduler/indexing_scheduler.rb

@@ -6,17 +6,19 @@ class Scheduler::IndexingScheduler
 
   sidekiq_options retry: 0
 
+  IMPORT_BATCH_SIZE = 1000
+  SCAN_BATCH_SIZE = 10 * IMPORT_BATCH_SIZE
+
   def perform
     return unless Chewy.enabled?
 
     indexes.each do |type|
       with_redis do |redis|
-        ids = redis.smembers("chewy:queue:#{type.name}")
-
-        type.import!(ids)
-
-        redis.pipelined do |pipeline|
-          ids.each { |id| pipeline.srem("chewy:queue:#{type.name}", id) }
+        redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids|
+          type.import!(ids)
+          redis.pipelined do |pipeline|
+            pipeline.srem("chewy:queue:#{type.name}", ids)
+          end
         end
       end
     end

app/workers/scheduler/user_cleanup_scheduler.rb

@@ -24,7 +24,7 @@ class Scheduler::UserCleanupScheduler
   def clean_discarded_statuses!
     Status.unscoped.discarded.where('deleted_at <= ?', 30.days.ago).find_in_batches do |statuses|
       RemovalWorker.push_bulk(statuses) do |status|
-        [status.id, { 'immediate' => true }]
+        [status.id, { 'immediate' => true, 'skip_streaming' => true }]
       end
     end
   end

app/workers/unfollow_follow_worker.rb

@@ -10,13 +10,7 @@ class UnfollowFollowWorker
     old_target_account = Account.find(old_target_account_id)
     new_target_account = Account.find(new_target_account_id)
 
-    follow    = follower_account.active_relationships.find_by(target_account: old_target_account)
-    reblogs   = follow&.show_reblogs?
-    notify    = follow&.notify?
-    languages = follow&.languages
-
-    FollowService.new.call(follower_account, new_target_account, reblogs: reblogs, notify: notify, languages: languages, bypass_locked: bypass_locked, bypass_limit: true)
-    UnfollowService.new.call(follower_account, old_target_account, skip_unmerge: true)
+    FollowMigrationService.new.call(follower_account, new_target_account, old_target_account, bypass_locked: bypass_locked)
   rescue ActiveRecord::RecordNotFound, Mastodon::NotPermittedError
     true
   end

bin/tootctl

@@ -5,7 +5,9 @@ require_relative '../config/boot'
 require_relative '../lib/cli'
 
 begin
-  Mastodon::CLI.start(ARGV)
+  Chewy.strategy(:mastodon) do
+    Mastodon::CLI.start(ARGV)
+  end
 rescue Interrupt
   exit(130)
 end

config/application.rb

@@ -29,6 +29,7 @@ require_relative '../lib/paperclip/url_generator_extensions'
 require_relative '../lib/paperclip/attachment_extensions'
 require_relative '../lib/paperclip/lazy_thumbnail'
 require_relative '../lib/paperclip/gif_transcoder'
+require_relative '../lib/paperclip/media_type_spoof_detector_extensions'
 require_relative '../lib/paperclip/transcoder'
 require_relative '../lib/paperclip/type_corrector'
 require_relative '../lib/paperclip/response_with_limit_adapter'
@@ -39,6 +40,7 @@ require_relative '../lib/mastodon/rack_middleware'
 require_relative '../lib/devise/two_factor_ldap_authenticatable'
 require_relative '../lib/devise/two_factor_pam_authenticatable'
 require_relative '../lib/chewy/strategy/mastodon'
+require_relative '../lib/chewy/strategy/bypass_with_warning'
 require_relative '../lib/webpacker/manifest_extensions'
 require_relative '../lib/webpacker/helper_extensions'
 require_relative '../lib/rails/engine_extensions'
@@ -159,6 +161,10 @@ module Mastodon
       end
     end
 
+    config.public_file_server.headers = {
+      'X-Content-Type-Options' => 'nosniff',
+    }
+
     # config.paths.add File.join('app', 'api'), glob: File.join('**', '*.rb')
     # config.autoload_paths += Dir[Rails.root.join('app', 'api', '*')]
 

config/database.yml

@@ -4,6 +4,7 @@ default: &default
   timeout: 5000
   encoding: unicode
   sslmode: <%= ENV['DB_SSLMODE'] || "prefer" %>
+  application_name: ''
 
 development:
   <<: *default

config/imagemagick/policy.xml

@@ -0,0 +1,27 @@
+<policymap>
+  <!-- Set some basic system resource limits -->
+  <policy domain="resource" name="time" value="60" />
+
+  <policy domain="module" rights="none" pattern="URL" />
+
+  <policy domain="filter" rights="none" pattern="*" />
+
+  <!--
+    Ideally, we would restrict ImageMagick to only accessing its own
+    disk-backed pixel cache as well as Mastodon-created Tempfiles.
+
+    However, those paths depend on the operating system and environment
+    variables, so they can only be known at runtime.
+
+    Furthermore, those paths are not necessarily shared across Mastodon
+    processes, so even creating a policy.xml at runtime is impractical.
+
+    For the time being, only disable indirect reads.
+  -->
+  <policy domain="path" rights="none" pattern="@*" />
+
+  <!-- Disallow any coder by default, and only enable ones required by Mastodon -->
+  <policy domain="coder" rights="none" pattern="*" />
+  <policy domain="coder" rights="read | write" pattern="{PNG,JPEG,GIF,HEIC,WEBP}" />
+  <policy domain="coder" rights="write" pattern="{HISTOGRAM,RGB,INFO}" />
+</policymap>

config/initializers/chewy.rb

@@ -19,7 +19,7 @@ Chewy.settings = {
 # cycle, which takes care of checking if Elasticsearch is enabled
 # or not. However, mind that for the Rails console, the :urgent
 # strategy is set automatically with no way to override it.
-Chewy.root_strategy              = :mastodon
+Chewy.root_strategy              = :bypass_with_warning if Rails.env.production?
 Chewy.request_strategy           = :mastodon
 Chewy.use_after_commit_callbacks = false
 

config/initializers/content_security_policy.rb

@@ -3,7 +3,7 @@
 # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy
 
 def host_to_url(str)
-  "http#{Rails.configuration.x.use_https ? 's' : ''}://#{str}" unless str.blank?
+  "http#{Rails.configuration.x.use_https ? 's' : ''}://#{str.split('/').first}" if str.present?
 end
 
 base_host = Rails.configuration.x.web_domain

config/initializers/paperclip.rb

@@ -124,6 +124,7 @@ elsif ENV['SWIFT_ENABLED'] == 'true'
       openstack_domain_name: ENV.fetch('SWIFT_DOMAIN_NAME') { 'default' },
       openstack_region: ENV['SWIFT_REGION'],
       openstack_cache_ttl: ENV.fetch('SWIFT_CACHE_TTL') { 60 },
+      openstack_temp_url_key: ENV['SWIFT_TEMP_URL_KEY'],
     },
     
     fog_file: { 'Cache-Control' => 'public, max-age=315576000, immutable' },
@@ -154,3 +155,10 @@ unless defined?(Seahorse)
     end
   end
 end
+
+# Set our ImageMagick security policy, but allow admins to override it
+ENV['MAGICK_CONFIGURE_PATH'] = begin
+  imagemagick_config_paths = ENV.fetch('MAGICK_CONFIGURE_PATH', '').split(File::PATH_SEPARATOR)
+  imagemagick_config_paths << Rails.root.join('config', 'imagemagick').expand_path.to_s
+  imagemagick_config_paths.join(File::PATH_SEPARATOR)
+end

config/initializers/sidekiq.rb

@@ -3,6 +3,11 @@
 require_relative '../../lib/mastodon/sidekiq_middleware'
 
 Sidekiq.configure_server do |config|
+  if Rails.configuration.database_configuration.dig('production', 'adapter') == 'postgresql_makara'
+    STDERR.puts 'ERROR: Database replication is not currently supported in Sidekiq workers. Check your configuration.'
+    exit 1
+  end
+
   config.redis = REDIS_SIDEKIQ_PARAMS
 
   config.server_middleware do |chain|

config/initializers/twitter_regex.rb

@@ -25,7 +25,7 @@ module Twitter::TwitterText
       \)
     /iox
     UCHARS = '\u{A0}-\u{D7FF}\u{F900}-\u{FDCF}\u{FDF0}-\u{FFEF}\u{10000}-\u{1FFFD}\u{20000}-\u{2FFFD}\u{30000}-\u{3FFFD}\u{40000}-\u{4FFFD}\u{50000}-\u{5FFFD}\u{60000}-\u{6FFFD}\u{70000}-\u{7FFFD}\u{80000}-\u{8FFFD}\u{90000}-\u{9FFFD}\u{A0000}-\u{AFFFD}\u{B0000}-\u{BFFFD}\u{C0000}-\u{CFFFD}\u{D0000}-\u{DFFFD}\u{E1000}-\u{EFFFD}\u{E000}-\u{F8FF}\u{F0000}-\u{FFFFD}\u{100000}-\u{10FFFD}'
-    REGEXEN[:valid_url_query_chars] = /[a-z0-9!?\*'\(\);:&=\+\$\/%#\[\]\-_\.,~|@#{UCHARS}]/iou
+    REGEXEN[:valid_url_query_chars] = /[a-z0-9!?\*'\(\);:&=\+\$\/%#\[\]\-_\.,~|@\^#{UCHARS}]/iou
     REGEXEN[:valid_url_query_ending_chars] = /[a-z0-9_&=#\/\-#{UCHARS}]/iou
     REGEXEN[:valid_url_path] = /(?:
       (?:

config/locales/activerecord.en.yml

@@ -53,3 +53,7 @@ en:
             position:
               elevated: cannot be higher than your current role
               own_role: cannot be changed with your current role
+        webhook:
+          attributes:
+            events:
+              invalid_permissions: cannot include events you don't have the rights to

config/locales/en.yml

@@ -756,6 +756,12 @@ en:
         message_html: You haven't defined any server rules.
       sidekiq_process_check:
         message_html: No Sidekiq process running for the %{value} queue(s). Please review your Sidekiq configuration
+      upload_check_privacy_error:
+        action: Check here for more information
+        message_html: "<strong>Your web server is misconfigured. The privacy of your users is at risk.</strong>"
+      upload_check_privacy_error_object_storage:
+        action: Check here for more information
+        message_html: "<strong>Your object storage is misconfigured. The privacy of your users is at risk.</strong>"
     tags:
       review: Review status
       updated_msg: Hashtag settings updated successfully
@@ -1326,6 +1332,7 @@ en:
       expired: The poll has already ended
       invalid_choice: The chosen vote option does not exist
       over_character_limit: cannot be longer than %{max} characters each
+      self_vote: You cannot vote in your own polls
       too_few_options: must have more than one item
       too_many_options: can't contain more than %{max} items
   preferences:
@@ -1341,6 +1348,7 @@ en:
   relationships:
     activity: Account activity
     dormant: Dormant
+    follow_failure: Could not follow some of the selected accounts.
     follow_selected_followers: Follow selected followers
     followers: Followers
     following: Following

config/routes.rb

@@ -109,6 +109,8 @@ Rails.application.routes.draw do
 
   resource :inbox, only: [:create], module: :activitypub
 
+  get '/:encoded_at(*path)', to: redirect("/@%{path}"), constraints: { encoded_at: /%40/ }
+
   constraints(username: /[^@\/.]+/) do
     get '/@:username', to: 'accounts#show', as: :short_account
     get '/@:username/with_replies', to: 'accounts#show', as: :short_account_with_replies
@@ -217,6 +219,7 @@ Rails.application.routes.draw do
   resource :statuses_cleanup, controller: :statuses_cleanup, only: [:show, :update]
 
   get '/media_proxy/:id/(*any)', to: 'media_proxy#show', as: :media_proxy, format: false
+  get '/backups/:id/download', to: 'backups#download', as: :download_backup, format: false
 
   resource :authorize_interaction, only: [:show, :create]
   resource :share, only: [:show, :create]
@@ -270,7 +273,7 @@ Rails.application.routes.draw do
       end
     end
 
-    resources :instances, only: [:index, :show, :destroy], constraints: { id: /[^\/]+/ } do
+    resources :instances, only: [:index, :show, :destroy], constraints: { id: /[^\/]+/ }, format: 'html' do
       member do
         post :clear_delivery_errors
         post :restart_delivery
@@ -447,7 +450,9 @@ Rails.application.routes.draw do
         resources :list, only: :show
       end
 
-      resources :streaming, only: [:index]
+      get '/streaming', to: 'streaming#index'
+      get '/streaming/(*any)', to: 'streaming#index'
+
       resources :custom_emojis, only: [:index]
       resources :suggestions, only: [:index, :destroy]
       resources :scheduled_statuses, only: [:index, :show, :update, :destroy]

db/seeds.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
-Dir[Rails.root.join('db', 'seeds', '*.rb')].sort.each do |seed|
-  load seed
+Chewy.strategy(:mastodon) do
+  Dir[Rails.root.join('db', 'seeds', '*.rb')].sort.each do |seed|
+    load seed
+  end
 end