Merge branch 'develop' into madlittlemods/17368-bust-_membership_stream_cache

.github/workflows/release-artifacts.yml

@@ -111,7 +111,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-12]
+        os: [ubuntu-22.04, macos-13]
         arch: [x86_64, aarch64]
         # is_pr is a flag used to exclude certain jobs from the matrix on PRs.
         # It is not read by the rest of the workflow.
@@ -121,9 +121,9 @@ jobs:
         exclude:
           # Don't build macos wheels on PR CI.
           - is_pr: true
-            os: "macos-12"
+            os: "macos-13"
           # Don't build aarch64 wheels on mac.
-          - os: "macos-12"
+          - os: "macos-13"
             arch: aarch64
           # Don't build aarch64 wheels on PR CI.
           - is_pr: true

CHANGES.md

@@ -1,6 +1,6 @@
-# Synapse 1.119.0rc2 (2024-11-11)
+# Synapse 1.119.0 (2024-11-13)
 
-Note that due to packaging issues there was no v1.119.0rc1.
+No significant changes since 1.119.0rc2.
 
 ### Python 3.8 support dropped
 
@@ -8,6 +8,12 @@ Python 3.8 is [end-of-life](https://devguide.python.org/versions/) and is no lon
 
 If you are running Synapse with Python 3.8, please upgrade to Python 3.9 (or greater) before upgrading Synapse.
 
+
+# Synapse 1.119.0rc2 (2024-11-11)
+
+Note that due to packaging issues there was no v1.119.0rc1.
+
+
 ### Features
 
 - Support [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151)'s stable report room API. ([\#17374](https://github.com/element-hq/synapse/issues/17374))

Cargo.lock

@@ -485,18 +485,18 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
 [[package]]
 name = "serde"
-version = "1.0.214"
+version = "1.0.215"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5"
+checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.214"
+version = "1.0.215"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766"
+checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0"
 dependencies = [
  "proc-macro2",
  "quote",

changelog.d/17638.removal

@@ -0,0 +1 @@
+Remove support for closed [MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886).

changelog.d/17924.misc

@@ -0,0 +1 @@
+Bump macos version used to build wheels during release, as current version used is end-of-life.

changelog.d/17928.misc

@@ -0,0 +1 @@
+Move server event filtering logic to rust.

changelog.d/17931.doc

@@ -0,0 +1 @@
+Add documentation about backing up Synapse.

debian/changelog

@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.119.0) stable; urgency=medium
+
+  * New Synapse release 1.119.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 13 Nov 2024 13:57:51 +0000
+
 matrix-synapse-py3 (1.119.0~rc2) stable; urgency=medium
 
   * New Synapse release 1.119.0rc2.

docs/SUMMARY.md

@@ -54,6 +54,7 @@
       - [Using `synctl` with Workers](synctl_workers.md)
       - [Systemd](systemd-with-workers/README.md)
   - [Administration](usage/administration/README.md)
+    - [Backups](usage/administration/backups.md)
     - [Admin API](usage/administration/admin_api/README.md)
       - [Account Validity](admin_api/account_validity.md)
       - [Background Updates](usage/administration/admin_api/background_updates.md)

docs/postgres.md

@@ -100,6 +100,10 @@ database:
     keepalives_count: 3
 ```
 
+## Backups
+
+Don't forget to [back up](./usage/administration/backups.md#database) your database!
+
 ## Tuning Postgres
 
 The default settings should be fine for most deployments. For larger

docs/setup/installation.md

@@ -656,6 +656,10 @@ This also requires the optional `lxml` python dependency to be  installed. This
 in turn requires the `libxml2` library to be available - on  Debian/Ubuntu this
 means `apt-get install libxml2-dev`, or equivalent for your OS.
 
+### Backups
+
+Don't forget to take [backups](../usage/administration/backups.md) of your new server!
+
 ### Troubleshooting Installation
 
 `pip` seems to leak *lots* of memory during installation. For instance, a Linux

docs/upgrade.md

@@ -117,6 +117,17 @@ each upgrade are complete before moving on to the next upgrade, to avoid
 stacking them up. You can monitor the currently running background updates with
 [the Admin API](usage/administration/admin_api/background_updates.html#status).
 
+# Upgrading to v1.120.0
+
+## Removal of experimental MSC3886 feature
+
+[MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886)
+has been closed (and will not enter the Matrix spec). As such, we are
+removing the experimental support for it in this release.
+
+The `experimental_features.msc3886_endpoint` configuration option has
+been removed.
+
 # Upgrading to v1.119.0
 
 ## Minimum supported Python version

docs/usage/administration/backups.md

@@ -0,0 +1,125 @@
+# How to back up a Synapse homeserver
+
+It is critical to maintain good backups of your server, to guard against
+hardware failure as well as potential corruption due to bugs or administrator
+error.
+
+This page documents the things you will need to consider backing up as part of
+a Synapse installation.
+
+## Configuration files
+
+Keep a copy of your configuration file (`homeserver.yaml`), as well as any
+auxiliary config files it refers to such as the
+[`log_config`](../configuration/config_documentation.md#log_config) file,
+[`app_service_config_files`](../configuration/config_documentation.md#app_service_config_files).
+Often, all such config files will be kept in a single directory such as
+`/etc/synapse`, which will make this easier.
+
+## Server signing key
+
+Your server has a [signing
+key](../configuration/config_documentation.md#signing_key_path) which it uses
+to sign events and outgoing federation requests. It is easiest to back it up
+with your configuration files, but an alternative is to have Synapse create a
+new signing key if you have to restore.
+
+If you do decide to replace the signing key, you should add the old *public*
+key to
+[`old_signing_keys`](../configuration/config_documentation.md#old_signing_keys).
+
+## Database
+
+Synapse's support for SQLite is only suitable for testing purposes, so for the
+purposes of this document, we'll assume you are using
+[PostgreSQL](../../postgres.md).
+
+A full discussion of backup strategies for PostgreSQL is out of scope for this
+document; see the [PostgreSQL
+documentation](https://www.postgresql.org/docs/current/backup.html) for
+detailed information.
+
+### Synapse-specfic details
+
+ * Be very careful not to restore into a database that already has tables
+   present. At best, this will error; at worst, it will lead to subtle database
+   inconsistencies.
+
+ * The `e2e_one_time_keys_json` table should **not** be backed up, or if it is
+   backed up, should be
+   [`TRUNCATE`d](https://www.postgresql.org/docs/current/sql-truncate.html)
+   after restoring the database before Synapse is started.
+
+   [Background: restoring the database to an older backup can cause
+   used one-time-keys to be re-issued, causing subsequent [message decryption
+   errors](https://github.com/element-hq/element-meta/issues/2155). Clearing
+   all one-time-keys from the database ensures that this cannot happen, and
+   will prompt clients to generate and upload new one-time-keys.]
+
+### Quick and easy database backup and restore
+
+Typically, the easiest solution is to use `pg_dump` to take a copy of the whole
+database. We recommend `pg_dump`'s custom dump format, as it produces
+significantly smaller backup files.
+
+```shell
+sudo -u postgres pg_dump -Fc --exclude-table-data e2e_one_time_keys_json synapse > synapse.dump
+```
+
+There is no need to stop Postgres or Synapse while `pg_dump` is running: it
+will take a consistent snapshot of the databse.
+
+To restore, you will need to recreate the database as described in [Using
+Postgres](../../postgres.md#set-up-database),
+then load the dump into it with `pg_restore`:
+
+```shell
+sudo -u postgres createdb --encoding=UTF8 --locale=C --template=template0 --owner=synapse_user synapse
+sudo -u postgres pg_restore -d synapse < synapse.dump
+```
+
+(If you forgot to exclude `e2e_one_time_keys_json` during `pg_dump`, remember
+to connect to the new database and `TRUNCATE e2e_one_time_keys_json;` before
+starting Synapse.)
+
+To reiterate: do **not** restore a dump over an existing database.
+
+Again, if you plan to run your homeserver at any sort of production level, we
+recommend studying the PostgreSQL documentation on backup options.
+
+## Media store
+
+Synapse keeps a copy of media uploaded by users, including avatars and message
+attachments, in its [Media
+store](../configuration/config_documentation.md#media-store).
+
+It is a directory on the local disk, containing the following directories:
+
+ * `local_content`: this is content uploaded by your local users. As a general
+   rule, you should back this up: it may represent the only copy of those
+   media files anywhere in the federation, and if they are lost, users will
+   see errors when viewing user or room avatars, and messages with attachments.
+
+ * `local_thumbnails`: "thumbnails" of images uploaded by your users. If
+   [`dynamic_thumbnails`](../configuration/config_documentation.md#dynamic_thumbnails)
+   is enabled, these will be regenerated if they are removed from the disk, and
+   there is therefore no need to back them up.
+
+   If `dynamic_thumbnails` is *not* enabled (the default): although this can
+   theoretically be regenerated from `local_content`, there is no tooling to do
+   so. We recommend that these are backed up too.
+
+ * `remote_content`: this is a cache of content that was uploaded by a user on
+    another server, and has since been requested by a user on your own server.
+
+    Typically there is no need to back up this directory: if a file in this directory
+    is removed, Synapse will attempt to fetch it again from the remote
+    server.
+
+ * `remote_thumbnails`: thumbnails of images uploaded by users on other
+    servers. As with `remote_content`, there is normally no need to back this
+    up.
+
+ * `url_cache`, `url_cache_thumbnails`: temporary caches of files downloaded
+   by the [URL previews](../../setup/installation.md#url-previews) feature.
+   These do not need to be backed up.

docs/usage/configuration/config_documentation.md

@@ -3128,6 +3128,15 @@ it was last used.
 It is possible to build an entry from an old `signing.key` file using the
 `export_signing_key` script which is provided with synapse.
 
+If you have lost the private key file, you can ask another server you trust to
+tell you the public keys it has seen from your server. To fetch the keys from
+`matrix.org`, try something like:
+
+```
+curl https://matrix-federation.matrix.org/_matrix/key/v2/query/myserver.example.com |
+  jq '.server_keys | map(.verify_keys) | add'
+```
+
 Example configuration:
 ```yaml
 old_signing_keys:

poetry.lock

@@ -724,13 +724,13 @@ files = [
 
 [[package]]
 name = "immutabledict"
-version = "4.2.0"
+version = "4.2.1"
 description = "Immutable wrapper around dictionaries (a fork of frozendict)"
 optional = false
-python-versions = ">=3.8,<4.0"
+python-versions = ">=3.8"
 files = [
-    {file = "immutabledict-4.2.0-py3-none-any.whl", hash = "sha256:d728b2c2410d698d95e6200237feb50a695584d20289ad3379a439aa3d90baba"},
+    {file = "immutabledict-4.2.1-py3-none-any.whl", hash = "sha256:c56a26ced38c236f79e74af3ccce53772827cef5c3bce7cab33ff2060f756373"},
-    {file = "immutabledict-4.2.0.tar.gz", hash = "sha256:e003fd81aad2377a5a758bf7e1086cf3b70b63e9a5cc2f46bce8d0a2b4727c5f"},
+    {file = "immutabledict-4.2.1.tar.gz", hash = "sha256:d91017248981c72eb66c8ff9834e99c2f53562346f23e7f51e7a5ebcf66a3bcc"},
 ]
 
 [[package]]
@@ -1418,13 +1418,13 @@ tests = ["Sphinx", "doubles", "flake8", "flake8-quotes", "gevent", "mock", "pyte
 
 [[package]]
 name = "packaging"
-version = "24.1"
+version = "24.2"
 description = "Core utilities for Python packages"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"},
+    {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"},
-    {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"},
+    {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"},
 ]
 
 [[package]]
@@ -1443,13 +1443,13 @@ dev = ["jinja2"]
 
 [[package]]
 name = "phonenumbers"
-version = "8.13.49"
+version = "8.13.50"
 description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers."
 optional = false
 python-versions = "*"
 files = [
-    {file = "phonenumbers-8.13.49-py2.py3-none-any.whl", hash = "sha256:e17140955ab3d8f9580727372ea64c5ada5327932d6021ef6fd203c3db8c8139"},
+    {file = "phonenumbers-8.13.50-py2.py3-none-any.whl", hash = "sha256:bb95dbc0d9979c51f7ad94bcd780784938958861fbb4b75a2fe39ccd3d58954a"},
-    {file = "phonenumbers-8.13.49.tar.gz", hash = "sha256:e608ccb61f0bd42e6db1d2c421f7c22186b88f494870bf40aa31d1a2718ab0ae"},
+    {file = "phonenumbers-8.13.50.tar.gz", hash = "sha256:e05ac6fb7b98c6d719a87ea895b9fc153673b4a51f455ec9afaf557ef4629da6"},
 ]
 
 [[package]]

pyproject.toml

@@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust"
 
 [tool.poetry]
 name = "matrix-synapse"
-version = "1.119.0rc2"
+version = "1.119.0"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "AGPL-3.0-or-later"

rust/src/events/filter.rs

@@ -0,0 +1,107 @@
+/*
+ * This file is licensed under the Affero General Public License (AGPL) version 3.
+ *
+ * Copyright (C) 2024 New Vector, Ltd
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * See the GNU Affero General Public License for more details:
+ * <https://www.gnu.org/licenses/agpl-3.0.html>.
+ */
+
+use std::collections::HashMap;
+
+use pyo3::{exceptions::PyValueError, pyfunction, PyResult};
+
+use crate::{
+    identifier::UserID,
+    matrix_const::{
+        HISTORY_VISIBILITY_INVITED, HISTORY_VISIBILITY_JOINED, MEMBERSHIP_INVITE, MEMBERSHIP_JOIN,
+    },
+};
+
+#[pyfunction(name = "event_visible_to_server")]
+pub fn event_visible_to_server_py(
+    sender: String,
+    target_server_name: String,
+    history_visibility: String,
+    erased_senders: HashMap<String, bool>,
+    partial_state_invisible: bool,
+    memberships: Vec<(String, String)>, // (state_key, membership)
+) -> PyResult<bool> {
+    event_visible_to_server(
+        sender,
+        target_server_name,
+        history_visibility,
+        erased_senders,
+        partial_state_invisible,
+        memberships,
+    )
+    .map_err(|e| PyValueError::new_err(format!("{e}")))
+}
+
+/// Return whether the target server is allowed to see the event.
+///
+/// For a fully stated room, the target server is allowed to see an event E if:
+///   - the state at E has world readable or shared history vis, OR
+///   - the state at E says that the target server is in the room.
+///
+/// For a partially stated room, the target server is allowed to see E if:
+///   - E was created by this homeserver, AND:
+///       - the partial state at E has world readable or shared history vis, OR
+///       - the partial state at E says that the target server is in the room.
+pub fn event_visible_to_server(
+    sender: String,
+    target_server_name: String,
+    history_visibility: String,
+    erased_senders: HashMap<String, bool>,
+    partial_state_invisible: bool,
+    memberships: Vec<(String, String)>, // (state_key, membership)
+) -> anyhow::Result<bool> {
+    if let Some(&erased) = erased_senders.get(&sender) {
+        if erased {
+            return Ok(false);
+        }
+    }
+
+    if partial_state_invisible {
+        return Ok(false);
+    }
+
+    if history_visibility != HISTORY_VISIBILITY_INVITED
+        && history_visibility != HISTORY_VISIBILITY_JOINED
+    {
+        return Ok(true);
+    }
+
+    let mut visible = false;
+    for (state_key, membership) in memberships {
+        let state_key = UserID::try_from(state_key.as_ref())
+            .map_err(|e| anyhow::anyhow!(format!("invalid user_id ({state_key}): {e}")))?;
+        if state_key.server_name() != target_server_name {
+            return Err(anyhow::anyhow!(
+                "state_key.server_name ({}) does not match target_server_name ({target_server_name})",
+                state_key.server_name()
+            ));
+        }
+
+        match membership.as_str() {
+            MEMBERSHIP_INVITE => {
+                if history_visibility == HISTORY_VISIBILITY_INVITED {
+                    visible = true;
+                    break;
+                }
+            }
+            MEMBERSHIP_JOIN => {
+                visible = true;
+                break;
+            }
+            _ => continue,
+        }
+    }
+
+    Ok(visible)
+}

rust/src/events/mod.rs

@@ -22,15 +22,17 @@
 
 use pyo3::{
     types::{PyAnyMethods, PyModule, PyModuleMethods},
-    Bound, PyResult, Python,
+    wrap_pyfunction, Bound, PyResult, Python,
 };
 
+pub mod filter;
 mod internal_metadata;
 
 /// Called when registering modules with python.
 pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
     let child_module = PyModule::new_bound(py, "events")?;
     child_module.add_class::<internal_metadata::EventInternalMetadata>()?;
+    child_module.add_function(wrap_pyfunction!(filter::event_visible_to_server_py, m)?)?;
 
     m.add_submodule(&child_module)?;
 

rust/src/identifier.rs

@@ -0,0 +1,86 @@
+/*
+ * This file is licensed under the Affero General Public License (AGPL) version 3.
+ *
+ * Copyright (C) 2024 New Vector, Ltd
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * See the GNU Affero General Public License for more details:
+ * <https://www.gnu.org/licenses/agpl-3.0.html>.
+ */
+
+//! # Matrix Identifiers
+//!
+//! This module contains definitions and utilities for working with matrix identifiers.
+
+use std::{fmt, ops::Deref};
+
+/// Errors that can occur when parsing a matrix identifier.
+#[derive(Clone, Debug, PartialEq)]
+pub enum IdentifierError {
+    IncorrectSigil,
+    MissingColon,
+}
+
+impl fmt::Display for IdentifierError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+/// A Matrix user_id.
+#[derive(Clone, Debug, PartialEq)]
+pub struct UserID(String);
+
+impl UserID {
+    /// Returns the `localpart` of the user_id.
+    pub fn localpart(&self) -> &str {
+        &self[1..self.colon_pos()]
+    }
+
+    /// Returns the `server_name` / `domain` of the user_id.
+    pub fn server_name(&self) -> &str {
+        &self[self.colon_pos() + 1..]
+    }
+
+    /// Returns the position of the ':' inside of the user_id.
+    /// Used when splitting the user_id into it's respective parts.
+    fn colon_pos(&self) -> usize {
+        self.find(':').unwrap()
+    }
+}
+
+impl TryFrom<&str> for UserID {
+    type Error = IdentifierError;
+
+    /// Will try creating a `UserID` from the provided `&str`.
+    /// Can fail if the user_id is incorrectly formatted.
+    fn try_from(s: &str) -> Result<Self, Self::Error> {
+        if !s.starts_with('@') {
+            return Err(IdentifierError::IncorrectSigil);
+        }
+
+        if s.find(':').is_none() {
+            return Err(IdentifierError::MissingColon);
+        }
+
+        Ok(UserID(s.to_string()))
+    }
+}
+
+impl Deref for UserID {
+    type Target = str;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl fmt::Display for UserID {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}

rust/src/lib.rs

@@ -6,6 +6,8 @@ pub mod acl;
 pub mod errors;
 pub mod events;
 pub mod http;
+pub mod identifier;
+pub mod matrix_const;
 pub mod push;
 pub mod rendezvous;
 

rust/src/matrix_const.rs

@@ -0,0 +1,28 @@
+/*
+ * This file is licensed under the Affero General Public License (AGPL) version 3.
+ *
+ * Copyright (C) 2024 New Vector, Ltd
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * See the GNU Affero General Public License for more details:
+ * <https://www.gnu.org/licenses/agpl-3.0.html>.
+ */
+
+//! # Matrix Constants
+//!
+//! This module contains definitions for constant values described by the matrix specification.
+
+pub const HISTORY_VISIBILITY_WORLD_READABLE: &str = "world_readable";
+pub const HISTORY_VISIBILITY_SHARED: &str = "shared";
+pub const HISTORY_VISIBILITY_INVITED: &str = "invited";
+pub const HISTORY_VISIBILITY_JOINED: &str = "joined";
+
+pub const MEMBERSHIP_BAN: &str = "ban";
+pub const MEMBERSHIP_LEAVE: &str = "leave";
+pub const MEMBERSHIP_KNOCK: &str = "knock";
+pub const MEMBERSHIP_INVITE: &str = "invite";
+pub const MEMBERSHIP_JOIN: &str = "join";

rust/src/push/utils.rs

@@ -23,7 +23,6 @@ use anyhow::bail;
 use anyhow::Context;
 use anyhow::Error;
 use lazy_static::lazy_static;
-use regex;
 use regex::Regex;
 use regex::RegexBuilder;
 

synapse/config/experimental.py

@@ -365,11 +365,6 @@ class ExperimentalConfig(Config):
         # MSC3874: Filtering /messages with rel_types / not_rel_types.
         self.msc3874_enabled: bool = experimental.get("msc3874_enabled", False)
 
-        # MSC3886: Simple client rendezvous capability
-        self.msc3886_endpoint: Optional[str] = experimental.get(
-            "msc3886_endpoint", None
-        )
-
         # MSC3890: Remotely silence local notifications
         # Note: This option requires "experimental_features.msc3391_enabled" to be
         # set to "true", in order to communicate account data deletions to clients.

synapse/config/server.py

@@ -215,9 +215,6 @@ class HttpListenerConfig:
     additional_resources: Dict[str, dict] = attr.Factory(dict)
     tag: Optional[str] = None
     request_id_header: Optional[str] = None
-    # If true, the listener will return CORS response headers compatible with MSC3886:
-    # https://github.com/matrix-org/matrix-spec-proposals/pull/3886
-    experimental_cors_msc3886: bool = False
 
 
 @attr.s(slots=True, frozen=True, auto_attribs=True)
@@ -1004,7 +1001,6 @@ def parse_listener_def(num: int, listener: Any) -> ListenerConfig:
             additional_resources=listener.get("additional_resources", {}),
             tag=listener.get("tag"),
             request_id_header=listener.get("request_id_header"),
-            experimental_cors_msc3886=listener.get("experimental_cors_msc3886", False),
         )
 
     if socket_path:

synapse/http/server.py

@@ -921,15 +921,6 @@ def set_cors_headers(request: "SynapseRequest") -> None:
             b"Access-Control-Expose-Headers",
             b"Synapse-Trace-Id, Server, ETag",
         )
-    elif request.experimental_cors_msc3886:
-        request.setHeader(
-            b"Access-Control-Allow-Headers",
-            b"X-Requested-With, Content-Type, Authorization, Date, If-Match, If-None-Match",
-        )
-        request.setHeader(
-            b"Access-Control-Expose-Headers",
-            b"ETag, Location, X-Max-Bytes",
-        )
     else:
         request.setHeader(
             b"Access-Control-Allow-Headers",

synapse/http/site.py

@@ -94,7 +94,6 @@ class SynapseRequest(Request):
         self.reactor = site.reactor
         self._channel = channel  # this is used by the tests
         self.start_time = 0.0
-        self.experimental_cors_msc3886 = site.experimental_cors_msc3886
 
         # The requester, if authenticated. For federation requests this is the
         # server name, for client requests this is the Requester object.
@@ -666,10 +665,6 @@ class SynapseSite(ProxySite):
 
         request_id_header = config.http_options.request_id_header
 
-        self.experimental_cors_msc3886: bool = (
-            config.http_options.experimental_cors_msc3886
-        )
-
         def request_factory(channel: HTTPChannel, queued: bool) -> Request:
             return request_class(
                 channel,

synapse/rest/client/rendezvous.py

@@ -34,51 +34,6 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-# n.b [MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886) has now been closed.
-# However, we want to keep this implementation around for some time.
-# TODO: define an end-of-life date for this implementation.
-class MSC3886RendezvousServlet(RestServlet):
-    """
-    This is a placeholder implementation of [MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886)
-    simple client rendezvous capability that is used by the "Sign in with QR" functionality.
-
-    This implementation only serves as a 307 redirect to a configured server rather than being a full implementation.
-
-    A module that implements the full functionality is available at: https://pypi.org/project/matrix-http-rendezvous-synapse/.
-
-    Request:
-
-    POST /rendezvous HTTP/1.1
-    Content-Type: ...
-
-    ...
-
-    Response:
-
-    HTTP/1.1 307
-    Location: <configured endpoint>
-    """
-
-    PATTERNS = client_patterns(
-        "/org.matrix.msc3886/rendezvous$", releases=[], v1=False, unstable=True
-    )
-
-    def __init__(self, hs: "HomeServer"):
-        super().__init__()
-        redirection_target: Optional[str] = hs.config.experimental.msc3886_endpoint
-        assert (
-            redirection_target is not None
-        ), "Servlet is only registered if there is a redirection target"
-        self.endpoint = redirection_target.encode("utf-8")
-
-    async def on_POST(self, request: SynapseRequest) -> None:
-        respond_with_redirect(
-            request, self.endpoint, statusCode=TEMPORARY_REDIRECT, cors=True
-        )
-
-    # PUT, GET and DELETE are not implemented as they should be fulfilled by the redirect target.
-
-
 class MSC4108DelegationRendezvousServlet(RestServlet):
     PATTERNS = client_patterns(
         "/org.matrix.msc4108/rendezvous$", releases=[], v1=False, unstable=True
@@ -114,9 +69,6 @@ class MSC4108RendezvousServlet(RestServlet):
 
 
 def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None:
-    if hs.config.experimental.msc3886_endpoint is not None:
-        MSC3886RendezvousServlet(hs).register(http_server)
-
     if hs.config.experimental.msc4108_enabled:
         MSC4108RendezvousServlet(hs).register(http_server)
 

synapse/rest/client/versions.py

@@ -149,9 +149,6 @@ class VersionsRestServlet(RestServlet):
                     "org.matrix.msc3881": msc3881_enabled,
                     # Adds support for filtering /messages by event relation.
                     "org.matrix.msc3874": self.config.experimental.msc3874_enabled,
-                    # Adds support for simple HTTP rendezvous as per MSC3886
-                    "org.matrix.msc3886": self.config.experimental.msc3886_endpoint
-                    is not None,
                     # Adds support for relation-based redactions as per MSC3912.
                     "org.matrix.msc3912": self.config.experimental.msc3912_enabled,
                     # Whether recursively provide relations is supported.

synapse/synapse_rust/events.pyi

@@ -10,7 +10,7 @@
 # See the GNU Affero General Public License for more details:
 # <https://www.gnu.org/licenses/agpl-3.0.html>.
 
-from typing import Optional
+from typing import List, Mapping, Optional, Tuple
 
 from synapse.types import JsonDict
 
@@ -105,3 +105,29 @@ class EventInternalMetadata:
 
     def is_notifiable(self) -> bool:
         """Whether this event can trigger a push notification"""
+
+def event_visible_to_server(
+    sender: str,
+    target_server_name: str,
+    history_visibility: str,
+    erased_senders: Mapping[str, bool],
+    partial_state_invisible: bool,
+    memberships: List[Tuple[str, str]],
+) -> bool:
+    """Determine whether the server is allowed to see the unredacted event.
+
+    Args:
+        sender: The sender of the event.
+        target_server_name: The server we want to send the event to.
+        history_visibility: The history_visibility value at the event.
+        erased_senders: A mapping of users and whether they have requested erasure. If a
+            user is not in the map, it is treated as though they haven't requested erasure.
+        partial_state_invisible: Whether the event should be treated as invisible due to
+            the partial state status of the room.
+        memberships: A list of membership state information at the event for users
+            matching the `target_server_name`. Each list item must contain a tuple of
+            (state_key, membership).
+
+    Returns:
+        Whether the server is allowed to see the unredacted event.
+    """

synapse/visibility.py

@@ -27,7 +27,6 @@ from typing import (
     Final,
     FrozenSet,
     List,
-    Mapping,
     Optional,
     Sequence,
     Set,
@@ -48,6 +47,7 @@ from synapse.events.utils import clone_event, prune_event
 from synapse.logging.opentracing import trace
 from synapse.storage.controllers import StorageControllers
 from synapse.storage.databases.main import DataStore
+from synapse.synapse_rust.events import event_visible_to_server
 from synapse.types import RetentionPolicy, StateMap, StrCollection, get_domain_from_id
 from synapse.types.state import StateFilter
 from synapse.util import Clock
@@ -628,17 +628,6 @@ async def filter_events_for_server(
     """Filter a list of events based on whether the target server is allowed to
     see them.
 
-    For a fully stated room, the target server is allowed to see an event E if:
-      - the state at E has world readable or shared history vis, OR
-      - the state at E says that the target server is in the room.
-
-    For a partially stated room, the target server is allowed to see E if:
-      - E was created by this homeserver, AND:
-          - the partial state at E has world readable or shared history vis, OR
-          - the partial state at E says that the target server is in the room.
-
-    TODO: state before or state after?
-
     Args:
         storage
         target_server_name
@@ -655,35 +644,6 @@ async def filter_events_for_server(
         The filtered events.
     """
 
-    def is_sender_erased(event: EventBase, erased_senders: Mapping[str, bool]) -> bool:
-        if erased_senders and erased_senders[event.sender]:
-            logger.info("Sender of %s has been erased, redacting", event.event_id)
-            return True
-        return False
-
-    def check_event_is_visible(
-        visibility: str, memberships: StateMap[EventBase]
-    ) -> bool:
-        if visibility not in (HistoryVisibility.INVITED, HistoryVisibility.JOINED):
-            return True
-
-        # We now loop through all membership events looking for
-        # membership states for the requesting server to determine
-        # if the server is either in the room or has been invited
-        # into the room.
-        for ev in memberships.values():
-            assert get_domain_from_id(ev.state_key) == target_server_name
-
-            memtype = ev.membership
-            if memtype == Membership.JOIN:
-                return True
-            elif memtype == Membership.INVITE:
-                if visibility == HistoryVisibility.INVITED:
-                    return True
-
-        # server has no users in the room: redact
-        return False
-
     if filter_out_erased_senders:
         erased_senders = await storage.main.are_users_erased(e.sender for e in events)
     else:
@@ -726,20 +686,16 @@ async def filter_events_for_server(
         target_server_name,
     )
 
-    def include_event_in_output(e: EventBase) -> bool:
-        erased = is_sender_erased(e, erased_senders)
-        visible = check_event_is_visible(
-            event_to_history_vis[e.event_id], event_to_memberships.get(e.event_id, {})
-        )
-
-        if e.event_id in partial_state_invisible_event_ids:
-            visible = False
-
-        return visible and not erased
-
     to_return = []
     for e in events:
-        if include_event_in_output(e):
+        if event_visible_to_server(
+            sender=e.sender,
+            target_server_name=target_server_name,
+            history_visibility=event_to_history_vis[e.event_id],
+            erased_senders=erased_senders,
+            partial_state_invisible=e.event_id in partial_state_invisible_event_ids,
+            memberships=list(event_to_memberships.get(e.event_id, {}).values()),
+        ):
             to_return.append(e)
         elif redact:
             to_return.append(prune_event(e))
@@ -796,7 +752,7 @@ async def _event_to_history_vis(
 
 async def _event_to_memberships(
     storage: StorageControllers, events: Collection[EventBase], server_name: str
-) -> Dict[str, StateMap[EventBase]]:
+) -> Dict[str, StateMap[Tuple[str, str]]]:
     """Get the remote membership list at each of the given events
 
     Returns a map from event id to state map, which will contain only membership events
@@ -849,7 +805,7 @@ async def _event_to_memberships(
 
     return {
         e_id: {
-            key: event_map[inner_e_id]
+            key: (event_map[inner_e_id].state_key, event_map[inner_e_id].membership)
             for key, inner_e_id in key_to_eid.items()
             if inner_e_id in event_map
         }

tests/logging/test_terse_json.py

@@ -164,7 +164,6 @@ class TerseJsonTestCase(LoggerCleanupMixin, TestCase):
         site.site_tag = "test-site"
         site.server_version_string = "Server v1"
         site.reactor = Mock()
-        site.experimental_cors_msc3886 = False
         request = SynapseRequest(
             cast(HTTPChannel, FakeChannel(site, self.reactor)), site
         )

tests/rest/client/test_rendezvous.py

@@ -34,7 +34,6 @@ from tests import unittest
 from tests.unittest import override_config
 from tests.utils import HAS_AUTHLIB
 
-msc3886_endpoint = "/_matrix/client/unstable/org.matrix.msc3886/rendezvous"
 msc4108_endpoint = "/_matrix/client/unstable/org.matrix.msc4108/rendezvous"
 
 
@@ -54,17 +53,9 @@ class RendezvousServletTestCase(unittest.HomeserverTestCase):
         }
 
     def test_disabled(self) -> None:
-        channel = self.make_request("POST", msc3886_endpoint, {}, access_token=None)
-        self.assertEqual(channel.code, 404)
         channel = self.make_request("POST", msc4108_endpoint, {}, access_token=None)
         self.assertEqual(channel.code, 404)
 
-    @override_config({"experimental_features": {"msc3886_endpoint": "/asd"}})
-    def test_msc3886_redirect(self) -> None:
-        channel = self.make_request("POST", msc3886_endpoint, {}, access_token=None)
-        self.assertEqual(channel.code, 307)
-        self.assertEqual(channel.headers.getRawHeaders("Location"), ["/asd"])
-
     @unittest.skip_unless(HAS_AUTHLIB, "requires authlib")
     @override_config(
         {

tests/server.py

@@ -343,7 +343,6 @@ class FakeSite:
         self,
         resource: IResource,
         reactor: IReactorTime,
-        experimental_cors_msc3886: bool = False,
     ):
         """
 
@@ -352,7 +351,6 @@ class FakeSite:
         """
         self._resource = resource
         self.reactor = reactor
-        self.experimental_cors_msc3886 = experimental_cors_msc3886
 
     def getResourceFor(self, request: Request) -> IResource:
         return self._resource

tests/test_server.py

@@ -233,9 +233,7 @@ class OptionsResourceTests(unittest.TestCase):
         self.resource = OptionsResource()
         self.resource.putChild(b"res", DummyResource())
 
-    def _make_request(
+    def _make_request(self, method: bytes, path: bytes) -> FakeChannel:
-        self, method: bytes, path: bytes, experimental_cors_msc3886: bool = False
-    ) -> FakeChannel:
         """Create a request from the method/path and return a channel with the response."""
         # Create a site and query for the resource.
         site = SynapseSite(
@@ -246,7 +244,6 @@ class OptionsResourceTests(unittest.TestCase):
                 {
                     "type": "http",
                     "port": 0,
-                    "experimental_cors_msc3886": experimental_cors_msc3886,
                 },
             ),
             self.resource,
@@ -283,32 +280,6 @@ class OptionsResourceTests(unittest.TestCase):
             [b"Synapse-Trace-Id, Server"],
         )
 
-    def _check_cors_msc3886_headers(self, channel: FakeChannel) -> None:
-        # Ensure the correct CORS headers have been added
-        # as per https://github.com/matrix-org/matrix-spec-proposals/blob/hughns/simple-rendezvous-capability/proposals/3886-simple-rendezvous-capability.md#cors
-        self.assertEqual(
-            channel.headers.getRawHeaders(b"Access-Control-Allow-Origin"),
-            [b"*"],
-            "has correct CORS Origin header",
-        )
-        self.assertEqual(
-            channel.headers.getRawHeaders(b"Access-Control-Allow-Methods"),
-            [b"GET, HEAD, POST, PUT, DELETE, OPTIONS"],  # HEAD isn't in the spec
-            "has correct CORS Methods header",
-        )
-        self.assertEqual(
-            channel.headers.getRawHeaders(b"Access-Control-Allow-Headers"),
-            [
-                b"X-Requested-With, Content-Type, Authorization, Date, If-Match, If-None-Match"
-            ],
-            "has correct CORS Headers header",
-        )
-        self.assertEqual(
-            channel.headers.getRawHeaders(b"Access-Control-Expose-Headers"),
-            [b"ETag, Location, X-Max-Bytes"],
-            "has correct CORS Expose Headers header",
-        )
-
     def test_unknown_options_request(self) -> None:
         """An OPTIONS requests to an unknown URL still returns 204 No Content."""
         channel = self._make_request(b"OPTIONS", b"/foo/")
@@ -325,16 +296,6 @@ class OptionsResourceTests(unittest.TestCase):
 
         self._check_cors_standard_headers(channel)
 
-    def test_known_options_request_msc3886(self) -> None:
-        """An OPTIONS requests to an known URL still returns 204 No Content."""
-        channel = self._make_request(
-            b"OPTIONS", b"/res/", experimental_cors_msc3886=True
-        )
-        self.assertEqual(channel.code, 204)
-        self.assertNotIn("body", channel.result)
-
-        self._check_cors_msc3886_headers(channel)
-
     def test_unknown_request(self) -> None:
         """A non-OPTIONS request to an unknown URL should 404."""
         channel = self._make_request(b"GET", b"/foo/")