From 03f0d746c39d2e7076f78da1edd6f0df316ee5d3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Mar 2024 11:49:11 +0000 Subject: [PATCH 1/5] 1.104.0rc1 --- CHANGES.md | 61 +++++++++++++++++++++++++++++++++++++++ changelog.d/16840.misc | 1 - changelog.d/16892.doc | 1 - changelog.d/16907.misc | 1 - changelog.d/16908.misc | 1 - changelog.d/16919.misc | 1 - changelog.d/16925.bugfix | 1 - changelog.d/16929.misc | 2 -- changelog.d/16949.bugfix | 1 - changelog.d/16950.misc | 1 - changelog.d/16953.misc | 1 - changelog.d/16965.doc | 1 - changelog.d/16966.doc | 1 - changelog.d/16971.feature | 1 - changelog.d/16972.feature | 1 - changelog.d/16974.misc | 1 - changelog.d/16978.docker | 1 - changelog.d/16985.misc | 1 - changelog.d/16986.misc | 1 - changelog.d/16990.bugfix | 1 - changelog.d/17002.doc | 1 - changelog.d/17010.bugfix | 1 - changelog.d/17017.misc | 1 - debian/changelog | 6 ++++ pyproject.toml | 2 +- 25 files changed, 68 insertions(+), 24 deletions(-) delete mode 100644 changelog.d/16840.misc delete mode 100644 changelog.d/16892.doc delete mode 100644 changelog.d/16907.misc delete mode 100644 changelog.d/16908.misc delete mode 100644 changelog.d/16919.misc delete mode 100644 changelog.d/16925.bugfix delete mode 100644 changelog.d/16929.misc delete mode 100644 changelog.d/16949.bugfix delete mode 100644 changelog.d/16950.misc delete mode 100644 changelog.d/16953.misc delete mode 100644 changelog.d/16965.doc delete mode 100644 changelog.d/16966.doc delete mode 100644 changelog.d/16971.feature delete mode 100644 changelog.d/16972.feature delete mode 100644 changelog.d/16974.misc delete mode 100644 changelog.d/16978.docker delete mode 100644 changelog.d/16985.misc delete mode 100644 changelog.d/16986.misc delete mode 100644 changelog.d/16990.bugfix delete mode 100644 changelog.d/17002.doc delete mode 100644 changelog.d/17010.bugfix delete mode 100644 changelog.d/17017.misc diff --git a/CHANGES.md b/CHANGES.md index c0679159df..ee658eda21 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,64 @@ +# Synapse 1.104.0rc1 (2024-03-26) + +### Features + +- Add an OIDC config to specify extra parameters for the authorization grant URL. IT can be useful to pass an ACR value for example. ([\#16971](https://github.com/element-hq/synapse/issues/16971)) +- OIDC: try to JWT decode userinfo response if JSON parsing failed. ([\#16972](https://github.com/element-hq/synapse/issues/16972)) + +### Bugfixes + +- Fix a bug which meant that, under certain circumstances, we might never retry sending events or to-device messages over federation after a failure. ([\#16925](https://github.com/element-hq/synapse/issues/16925)) +- Fix various long-standing bugs which could cause incorrect state to be returned from `/sync` in certain situations. ([\#16949](https://github.com/element-hq/synapse/issues/16949)) +- Fix case in which `m.fully_read` marker would not get updated. Contributed by @SpiritCroc. ([\#16990](https://github.com/element-hq/synapse/issues/16990)) +- Fix bug which did not retract a user's pending knocks at rooms when their account was deactivated. Contributed by @hanadi92. ([\#17010](https://github.com/element-hq/synapse/issues/17010)) + +### Updates to the Docker image + +- Updated start.py to generate config using the correct user ID when running as root (fixes #16824, #15202). ([\#16978](https://github.com/element-hq/synapse/issues/16978)) + +### Improved Documentation + +- Add a query to force a refresh of a remote user's device list to the "Useful SQL for Admins" documentation page. ([\#16892](https://github.com/element-hq/synapse/issues/16892)) +- Minor grammatical corrections to the upgrade documentation. ([\#16965](https://github.com/element-hq/synapse/issues/16965)) +- Fix the sort order for the documentation version picker, so that newer releases appear above older ones. ([\#16966](https://github.com/element-hq/synapse/issues/16966)) +- Remove recommendation for a specific poetry version from contributing guide. ([\#17002](https://github.com/element-hq/synapse/issues/17002)) + +### Internal Changes + +- Improve lock performance when a lot of locks are all waiting for a single lock to be released. ([\#16840](https://github.com/element-hq/synapse/issues/16840)) +- Update power level default for public rooms (#16907). ([\#16907](https://github.com/element-hq/synapse/issues/16907)) +- Improve event validation (#16908). ([\#16908](https://github.com/element-hq/synapse/issues/16908)) +- Multi-worker-docker-container: disable log buffering. ([\#16919](https://github.com/element-hq/synapse/issues/16919)) +- Refactor state delta calculation in `/sync` handler. ([\#16929](https://github.com/element-hq/synapse/issues/16929)) +- Clarify docs for some room state functions. ([\#16950](https://github.com/element-hq/synapse/issues/16950)) +- Specify IP subnets in canonical form. ([\#16953](https://github.com/element-hq/synapse/issues/16953)) +- As done for SAML mapping provider, let's pass the module API to the OIDC one so the mapper can do more logic in its code. ([\#16974](https://github.com/element-hq/synapse/issues/16974)) +- Allow containers building on top of Synapse's Complement container is use the included PostgreSQL cluster. ([\#16985](https://github.com/element-hq/synapse/issues/16985)) +- Raise poetry-core version cap to 1.9.0. ([\#16986](https://github.com/element-hq/synapse/issues/16986)) +- Patch the db conn pool sooner in tests. ([\#17017](https://github.com/element-hq/synapse/issues/17017)) + + + +### Updates to locked dependencies + +* Bump anyhow from 1.0.80 to 1.0.81. ([\#17009](https://github.com/element-hq/synapse/issues/17009)) +* Bump black from 23.10.1 to 24.2.0. ([\#16936](https://github.com/element-hq/synapse/issues/16936)) +* Bump cryptography from 41.0.7 to 42.0.5. ([\#16958](https://github.com/element-hq/synapse/issues/16958)) +* Bump dawidd6/action-download-artifact from 3.1.1 to 3.1.2. ([\#16960](https://github.com/element-hq/synapse/issues/16960)) +* Bump dawidd6/action-download-artifact from 3.1.2 to 3.1.4. ([\#17008](https://github.com/element-hq/synapse/issues/17008)) +* Bump jinja2 from 3.1.2 to 3.1.3. ([\#17005](https://github.com/element-hq/synapse/issues/17005)) +* Bump log from 0.4.20 to 0.4.21. ([\#16977](https://github.com/element-hq/synapse/issues/16977)) +* Bump mypy from 1.5.1 to 1.8.0. ([\#16901](https://github.com/element-hq/synapse/issues/16901)) +* Bump netaddr from 0.9.0 to 1.2.1. ([\#17006](https://github.com/element-hq/synapse/issues/17006)) +* Bump pydantic from 2.6.0 to 2.6.4. ([\#17004](https://github.com/element-hq/synapse/issues/17004)) +* Bump pyo3 from 0.20.2 to 0.20.3. ([\#16962](https://github.com/element-hq/synapse/issues/16962)) +* Bump ruff from 0.1.14 to 0.3.2. ([\#16994](https://github.com/element-hq/synapse/issues/16994)) +* Bump serde from 1.0.196 to 1.0.197. ([\#16963](https://github.com/element-hq/synapse/issues/16963)) +* Bump serde_json from 1.0.113 to 1.0.114. ([\#16961](https://github.com/element-hq/synapse/issues/16961)) +* Bump types-jsonschema from 4.21.0.20240118 to 4.21.0.20240311. ([\#17007](https://github.com/element-hq/synapse/issues/17007)) +* Bump types-psycopg2 from 2.9.21.16 to 2.9.21.20240311. ([\#16995](https://github.com/element-hq/synapse/issues/16995)) +* Bump types-pyopenssl from 23.3.0.0 to 24.0.0.20240311. ([\#17003](https://github.com/element-hq/synapse/issues/17003)) + # Synapse 1.103.0 (2024-03-19) No significant changes since 1.103.0rc1. diff --git a/changelog.d/16840.misc b/changelog.d/16840.misc deleted file mode 100644 index 1175e6de71..0000000000 --- a/changelog.d/16840.misc +++ /dev/null @@ -1 +0,0 @@ -Improve lock performance when a lot of locks are all waiting for a single lock to be released. diff --git a/changelog.d/16892.doc b/changelog.d/16892.doc deleted file mode 100644 index dd82b49112..0000000000 --- a/changelog.d/16892.doc +++ /dev/null @@ -1 +0,0 @@ -Add a query to force a refresh of a remote user's device list to the "Useful SQL for Admins" documentation page. \ No newline at end of file diff --git a/changelog.d/16907.misc b/changelog.d/16907.misc deleted file mode 100644 index e3a7fad447..0000000000 --- a/changelog.d/16907.misc +++ /dev/null @@ -1 +0,0 @@ -Update power level default for public rooms (#16907). \ No newline at end of file diff --git a/changelog.d/16908.misc b/changelog.d/16908.misc deleted file mode 100644 index d13c59aa35..0000000000 --- a/changelog.d/16908.misc +++ /dev/null @@ -1 +0,0 @@ -Improve event validation (#16908). \ No newline at end of file diff --git a/changelog.d/16919.misc b/changelog.d/16919.misc deleted file mode 100644 index 2c76f25379..0000000000 --- a/changelog.d/16919.misc +++ /dev/null @@ -1 +0,0 @@ -Multi-worker-docker-container: disable log buffering. diff --git a/changelog.d/16925.bugfix b/changelog.d/16925.bugfix deleted file mode 100644 index 781cad4b5b..0000000000 --- a/changelog.d/16925.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug which meant that, under certain circumstances, we might never retry sending events or to-device messages over federation after a failure. diff --git a/changelog.d/16929.misc b/changelog.d/16929.misc deleted file mode 100644 index 9489784e4a..0000000000 --- a/changelog.d/16929.misc +++ /dev/null @@ -1,2 +0,0 @@ -Refactor state delta calculation in `/sync` handler. - diff --git a/changelog.d/16949.bugfix b/changelog.d/16949.bugfix deleted file mode 100644 index 99ed435d75..0000000000 --- a/changelog.d/16949.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix various long-standing bugs which could cause incorrect state to be returned from `/sync` in certain situations. diff --git a/changelog.d/16950.misc b/changelog.d/16950.misc deleted file mode 100644 index c5a0741c3b..0000000000 --- a/changelog.d/16950.misc +++ /dev/null @@ -1 +0,0 @@ -Clarify docs for some room state functions. diff --git a/changelog.d/16953.misc b/changelog.d/16953.misc deleted file mode 100644 index 81abcc3df0..0000000000 --- a/changelog.d/16953.misc +++ /dev/null @@ -1 +0,0 @@ -Specify IP subnets in canonical form. diff --git a/changelog.d/16965.doc b/changelog.d/16965.doc deleted file mode 100644 index 36f8093298..0000000000 --- a/changelog.d/16965.doc +++ /dev/null @@ -1 +0,0 @@ -Minor grammatical corrections to the upgrade documentation. diff --git a/changelog.d/16966.doc b/changelog.d/16966.doc deleted file mode 100644 index 06f4093aee..0000000000 --- a/changelog.d/16966.doc +++ /dev/null @@ -1 +0,0 @@ -Fix the sort order for the documentation version picker, so that newer releases appear above older ones. diff --git a/changelog.d/16971.feature b/changelog.d/16971.feature deleted file mode 100644 index 9fdc88a322..0000000000 --- a/changelog.d/16971.feature +++ /dev/null @@ -1 +0,0 @@ -Add an OIDC config to specify extra parameters for the authorization grant URL. IT can be useful to pass an ACR value for example. diff --git a/changelog.d/16972.feature b/changelog.d/16972.feature deleted file mode 100644 index 0f28cbbcd6..0000000000 --- a/changelog.d/16972.feature +++ /dev/null @@ -1 +0,0 @@ -OIDC: try to JWT decode userinfo response if JSON parsing failed. diff --git a/changelog.d/16974.misc b/changelog.d/16974.misc deleted file mode 100644 index bf0a13786c..0000000000 --- a/changelog.d/16974.misc +++ /dev/null @@ -1 +0,0 @@ -As done for SAML mapping provider, let's pass the module API to the OIDC one so the mapper can do more logic in its code. diff --git a/changelog.d/16978.docker b/changelog.d/16978.docker deleted file mode 100644 index 2e095c6aa9..0000000000 --- a/changelog.d/16978.docker +++ /dev/null @@ -1 +0,0 @@ -Updated start.py to generate config using the correct user ID when running as root (fixes #16824, #15202). diff --git a/changelog.d/16985.misc b/changelog.d/16985.misc deleted file mode 100644 index 34d1337a28..0000000000 --- a/changelog.d/16985.misc +++ /dev/null @@ -1 +0,0 @@ -Allow containers building on top of Synapse's Complement container is use the included PostgreSQL cluster. diff --git a/changelog.d/16986.misc b/changelog.d/16986.misc deleted file mode 100644 index 0a556ba8b4..0000000000 --- a/changelog.d/16986.misc +++ /dev/null @@ -1 +0,0 @@ -Raise poetry-core version cap to 1.9.0. diff --git a/changelog.d/16990.bugfix b/changelog.d/16990.bugfix deleted file mode 100644 index 76f9dd2e36..0000000000 --- a/changelog.d/16990.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix case in which `m.fully_read` marker would not get updated. Contributed by @SpiritCroc. diff --git a/changelog.d/17002.doc b/changelog.d/17002.doc deleted file mode 100644 index a1b2ce5e31..0000000000 --- a/changelog.d/17002.doc +++ /dev/null @@ -1 +0,0 @@ -Remove recommendation for a specific poetry version from contributing guide. diff --git a/changelog.d/17010.bugfix b/changelog.d/17010.bugfix deleted file mode 100644 index 0e1495f744..0000000000 --- a/changelog.d/17010.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug which did not retract a user's pending knocks at rooms when their account was deactivated. Contributed by @hanadi92. \ No newline at end of file diff --git a/changelog.d/17017.misc b/changelog.d/17017.misc deleted file mode 100644 index c8af23d67a..0000000000 --- a/changelog.d/17017.misc +++ /dev/null @@ -1 +0,0 @@ -Patch the db conn pool sooner in tests. diff --git a/debian/changelog b/debian/changelog index c718dec7b0..b915b6e2cb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.104.0~rc1) stable; urgency=medium + + * New Synapse release 1.104.0rc1. + + -- Synapse Packaging team Tue, 26 Mar 2024 11:48:58 +0000 + matrix-synapse-py3 (1.103.0) stable; urgency=medium * New Synapse release 1.103.0. diff --git a/pyproject.toml b/pyproject.toml index 3bd5736755..8369139301 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,7 +96,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.103.0" +version = "1.104.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 05817413425c9b92e3f7a463221e628c4ceedbed Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Mar 2024 13:44:06 +0000 Subject: [PATCH 2/5] Fixup changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index ee658eda21..0bbacd393e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,7 +3,7 @@ ### Features - Add an OIDC config to specify extra parameters for the authorization grant URL. IT can be useful to pass an ACR value for example. ([\#16971](https://github.com/element-hq/synapse/issues/16971)) -- OIDC: try to JWT decode userinfo response if JSON parsing failed. ([\#16972](https://github.com/element-hq/synapse/issues/16972)) +- Add support for OIDC provider returning JWT. ([\#16972](https://github.com/element-hq/synapse/issues/16972), [\#17031](https://github.com/element-hq/synapse/issues/17031)) ### Bugfixes From 59ceabcb9798793cd4312fdbcced4e612aeda84d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Mar 2024 13:45:57 +0000 Subject: [PATCH 3/5] Fixup changelog --- CHANGES.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0bbacd393e..fa9af218a6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,7 +14,7 @@ ### Updates to the Docker image -- Updated start.py to generate config using the correct user ID when running as root (fixes #16824, #15202). ([\#16978](https://github.com/element-hq/synapse/issues/16978)) +- Updated `start.py` to generate config using the correct user ID when running as root (fixes [\#16824](https://github.com/element-hq/synapse/issues/16824), [\#15202](https://github.com/element-hq/synapse/issues/15202)). ([\#16978](https://github.com/element-hq/synapse/issues/16978)) ### Improved Documentation @@ -26,8 +26,8 @@ ### Internal Changes - Improve lock performance when a lot of locks are all waiting for a single lock to be released. ([\#16840](https://github.com/element-hq/synapse/issues/16840)) -- Update power level default for public rooms (#16907). ([\#16907](https://github.com/element-hq/synapse/issues/16907)) -- Improve event validation (#16908). ([\#16908](https://github.com/element-hq/synapse/issues/16908)) +- Update power level default for public rooms. ([\#16907](https://github.com/element-hq/synapse/issues/16907)) +- Improve event validation. ([\#16908](https://github.com/element-hq/synapse/issues/16908)) - Multi-worker-docker-container: disable log buffering. ([\#16919](https://github.com/element-hq/synapse/issues/16919)) - Refactor state delta calculation in `/sync` handler. ([\#16929](https://github.com/element-hq/synapse/issues/16929)) - Clarify docs for some room state functions. ([\#16950](https://github.com/element-hq/synapse/issues/16950)) From ea6bfae0fca5303bcf2e474694d6a388ef3b6a90 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Mar 2024 15:44:07 +0000 Subject: [PATCH 4/5] Add support for moving `/push_rules` off of main process (#17037) --- changelog.d/17037.feature | 1 + docs/workers.md | 7 +++ synapse/_scripts/synapse_port_db.py | 9 +-- synapse/config/workers.py | 12 ++++ synapse/handlers/room_member.py | 19 +++++- synapse/replication/http/push.py | 41 +++++++++++++ synapse/replication/tcp/handler.py | 7 +++ synapse/rest/client/push_rule.py | 6 +- synapse/storage/databases/main/__init__.py | 4 +- synapse/storage/databases/main/push_rule.py | 67 +++++++++++++-------- 10 files changed, 133 insertions(+), 40 deletions(-) create mode 100644 changelog.d/17037.feature diff --git a/changelog.d/17037.feature b/changelog.d/17037.feature new file mode 100644 index 0000000000..bd419c817d --- /dev/null +++ b/changelog.d/17037.feature @@ -0,0 +1 @@ +Add support for moving `/push_rules` off of main process. diff --git a/docs/workers.md b/docs/workers.md index d19f1a9dea..5ea8ad59bd 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -532,6 +532,13 @@ the stream writer for the `presence` stream: ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/ +##### The `push` stream + +The following endpoints should be routed directly to the worker configured as +the stream writer for the `push` stream: + + ^/_matrix/client/(api/v1|r0|v3|unstable)/push_rules/ + #### Restrict outbound federation traffic to a specific set of workers The diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index a533cad5ae..15507372a4 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -60,7 +60,7 @@ from synapse.logging.context import ( ) from synapse.notifier import ReplicationNotifier from synapse.storage.database import DatabasePool, LoggingTransaction, make_conn -from synapse.storage.databases.main import FilteringWorkerStore, PushRuleStore +from synapse.storage.databases.main import FilteringWorkerStore from synapse.storage.databases.main.account_data import AccountDataWorkerStore from synapse.storage.databases.main.client_ips import ClientIpBackgroundUpdateStore from synapse.storage.databases.main.deviceinbox import DeviceInboxBackgroundUpdateStore @@ -77,10 +77,8 @@ from synapse.storage.databases.main.media_repository import ( ) from synapse.storage.databases.main.presence import PresenceBackgroundUpdateStore from synapse.storage.databases.main.profile import ProfileWorkerStore -from synapse.storage.databases.main.pusher import ( - PusherBackgroundUpdatesStore, - PusherWorkerStore, -) +from synapse.storage.databases.main.push_rule import PusherWorkerStore +from synapse.storage.databases.main.pusher import PusherBackgroundUpdatesStore from synapse.storage.databases.main.receipts import ReceiptsBackgroundUpdateStore from synapse.storage.databases.main.registration import ( RegistrationBackgroundUpdateStore, @@ -245,7 +243,6 @@ class Store( AccountDataWorkerStore, FilteringWorkerStore, ProfileWorkerStore, - PushRuleStore, PusherWorkerStore, PusherBackgroundUpdatesStore, PresenceBackgroundUpdateStore, diff --git a/synapse/config/workers.py b/synapse/config/workers.py index e9c67807e5..9f81a73d6f 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -156,6 +156,8 @@ class WriterLocations: can only be a single instance. presence: The instances that write to the presence stream. Currently can only be a single instance. + push: The instances that write to the push stream. Currently + can only be a single instance. """ events: List[str] = attr.ib( @@ -182,6 +184,10 @@ class WriterLocations: default=["master"], converter=_instance_to_list_converter, ) + push: List[str] = attr.ib( + default=["master"], + converter=_instance_to_list_converter, + ) @attr.s(auto_attribs=True) @@ -341,6 +347,7 @@ class WorkerConfig(Config): "account_data", "receipts", "presence", + "push", ): instances = _instance_to_list_converter(getattr(self.writers, stream)) for instance in instances: @@ -378,6 +385,11 @@ class WorkerConfig(Config): "Must only specify one instance to handle `presence` messages." ) + if len(self.writers.push) != 1: + raise ConfigError( + "Must only specify one instance to handle `push` messages." + ) + self.events_shard_config = RoutableShardedWorkerHandlingConfig( self.writers.events ) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 9e9f6cd062..ee2e807afc 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -51,6 +51,7 @@ from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging import opentracing from synapse.metrics import event_processing_positions from synapse.metrics.background_process_metrics import run_as_background_process +from synapse.replication.http.push import ReplicationCopyPusherRestServlet from synapse.storage.databases.main.state_deltas import StateDelta from synapse.types import ( JsonDict, @@ -181,6 +182,10 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): hs.config.server.forgotten_room_retention_period ) + self._is_push_writer = hs.get_instance_name() in hs.config.worker.writers.push + self._push_writer = hs.config.worker.writers.push[0] + self._copy_push_client = ReplicationCopyPusherRestServlet.make_client(hs) + def _on_user_joined_room(self, event_id: str, room_id: str) -> None: """Notify the rate limiter that a room join has occurred. @@ -1301,9 +1306,17 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): old_room_id, new_room_id, user_id ) # Copy over push rules - await self.store.copy_push_rules_from_room_to_room_for_user( - old_room_id, new_room_id, user_id - ) + if self._is_push_writer: + await self.store.copy_push_rules_from_room_to_room_for_user( + old_room_id, new_room_id, user_id + ) + else: + await self._copy_push_client( + instance_name=self._push_writer, + user_id=user_id, + old_room_id=old_room_id, + new_room_id=new_room_id, + ) except Exception: logger.exception( "Error copying tags and/or push rules from rooms %s to %s for user %s. " diff --git a/synapse/replication/http/push.py b/synapse/replication/http/push.py index 8e5641707a..de07e75b46 100644 --- a/synapse/replication/http/push.py +++ b/synapse/replication/http/push.py @@ -77,5 +77,46 @@ class ReplicationRemovePusherRestServlet(ReplicationEndpoint): return 200, {} +class ReplicationCopyPusherRestServlet(ReplicationEndpoint): + """Copies push rules from an old room to new room. + + Request format: + + POST /_synapse/replication/copy_push_rules/:user_id/:old_room_id/:new_room_id + + {} + + """ + + NAME = "copy_push_rules" + PATH_ARGS = ("user_id", "old_room_id", "new_room_id") + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self._store = hs.get_datastores().main + + @staticmethod + async def _serialize_payload(user_id: str, old_room_id: str, new_room_id: str) -> JsonDict: # type: ignore[override] + return {} + + async def _handle_request( # type: ignore[override] + self, + request: Request, + content: JsonDict, + user_id: str, + old_room_id: str, + new_room_id: str, + ) -> Tuple[int, JsonDict]: + + await self._store.copy_push_rules_from_room_to_room_for_user( + old_room_id, new_room_id, user_id + ) + + return 200, {} + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: ReplicationRemovePusherRestServlet(hs).register(http_server) + ReplicationCopyPusherRestServlet(hs).register(http_server) diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index ecc12c0b28..4342d6ce70 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -66,6 +66,7 @@ from synapse.replication.tcp.streams import ( FederationStream, PresenceFederationStream, PresenceStream, + PushRulesStream, ReceiptsStream, Stream, ToDeviceStream, @@ -178,6 +179,12 @@ class ReplicationCommandHandler: continue + if isinstance(stream, PushRulesStream): + if hs.get_instance_name() in hs.config.worker.writers.push: + self._streams_to_replicate.append(stream) + + continue + # Only add any other streams if we're on master. if hs.config.worker.worker_app is not None: continue diff --git a/synapse/rest/client/push_rule.py b/synapse/rest/client/push_rule.py index 7d58611abb..9c9bfc9794 100644 --- a/synapse/rest/client/push_rule.py +++ b/synapse/rest/client/push_rule.py @@ -59,12 +59,12 @@ class PushRuleRestServlet(RestServlet): self.auth = hs.get_auth() self.store = hs.get_datastores().main self.notifier = hs.get_notifier() - self._is_worker = hs.config.worker.worker_app is not None + self._is_push_worker = hs.get_instance_name() in hs.config.worker.writers.push self._push_rules_handler = hs.get_push_rules_handler() self._push_rule_linearizer = Linearizer(name="push_rules") async def on_PUT(self, request: SynapseRequest, path: str) -> Tuple[int, JsonDict]: - if self._is_worker: + if not self._is_push_worker: raise Exception("Cannot handle PUT /push_rules on worker") requester = await self.auth.get_user_by_req(request) @@ -137,7 +137,7 @@ class PushRuleRestServlet(RestServlet): async def on_DELETE( self, request: SynapseRequest, path: str ) -> Tuple[int, JsonDict]: - if self._is_worker: + if not self._is_push_worker: raise Exception("Cannot handle DELETE /push_rules on worker") requester = await self.auth.get_user_by_req(request) diff --git a/synapse/storage/databases/main/__init__.py b/synapse/storage/databases/main/__init__.py index bf779587d9..586e84f2a4 100644 --- a/synapse/storage/databases/main/__init__.py +++ b/synapse/storage/databases/main/__init__.py @@ -63,7 +63,7 @@ from .openid import OpenIdStore from .presence import PresenceStore from .profile import ProfileStore from .purge_events import PurgeEventsStore -from .push_rule import PushRuleStore +from .push_rule import PushRulesWorkerStore from .pusher import PusherStore from .receipts import ReceiptsStore from .registration import RegistrationStore @@ -130,7 +130,6 @@ class DataStore( RejectionsStore, FilteringWorkerStore, PusherStore, - PushRuleStore, ApplicationServiceTransactionStore, EventPushActionsStore, ServerMetricsStore, @@ -140,6 +139,7 @@ class DataStore( SearchStore, TagsStore, AccountDataStore, + PushRulesWorkerStore, StreamWorkerStore, OpenIdStore, ClientIpWorkerStore, diff --git a/synapse/storage/databases/main/push_rule.py b/synapse/storage/databases/main/push_rule.py index 91beca6ffc..ed734f03ac 100644 --- a/synapse/storage/databases/main/push_rule.py +++ b/synapse/storage/databases/main/push_rule.py @@ -53,11 +53,7 @@ from synapse.storage.databases.main.receipts import ReceiptsWorkerStore from synapse.storage.databases.main.roommember import RoomMemberWorkerStore from synapse.storage.engines import PostgresEngine, Sqlite3Engine from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException -from synapse.storage.util.id_generators import ( - AbstractStreamIdGenerator, - IdGenerator, - StreamIdGenerator, -) +from synapse.storage.util.id_generators import IdGenerator, StreamIdGenerator from synapse.synapse_rust.push import FilteredPushRules, PushRule, PushRules from synapse.types import JsonDict from synapse.util import json_encoder, unwrapFirstError @@ -130,6 +126,8 @@ class PushRulesWorkerStore( `get_max_push_rules_stream_id` which can be called in the initializer. """ + _push_rules_stream_id_gen: StreamIdGenerator + def __init__( self, database: DatabasePool, @@ -138,6 +136,8 @@ class PushRulesWorkerStore( ): super().__init__(database, db_conn, hs) + self._is_push_writer = hs.get_instance_name() in hs.config.worker.writers.push + # In the worker store this is an ID tracker which we overwrite in the non-worker # class below that is used on the main process. self._push_rules_stream_id_gen = StreamIdGenerator( @@ -145,7 +145,7 @@ class PushRulesWorkerStore( hs.get_replication_notifier(), "push_rules_stream", "stream_id", - is_writer=hs.config.worker.worker_app is None, + is_writer=self._is_push_writer, ) push_rules_prefill, push_rules_id = self.db_pool.get_cache_dict( @@ -162,6 +162,9 @@ class PushRulesWorkerStore( prefilled_cache=push_rules_prefill, ) + self._push_rule_id_gen = IdGenerator(db_conn, "push_rules", "id") + self._push_rules_enable_id_gen = IdGenerator(db_conn, "push_rules_enable", "id") + def get_max_push_rules_stream_id(self) -> int: """Get the position of the push rules stream. @@ -383,23 +386,6 @@ class PushRulesWorkerStore( "get_all_push_rule_updates", get_all_push_rule_updates_txn ) - -class PushRuleStore(PushRulesWorkerStore): - # Because we have write access, this will be a StreamIdGenerator - # (see PushRulesWorkerStore.__init__) - _push_rules_stream_id_gen: AbstractStreamIdGenerator - - def __init__( - self, - database: DatabasePool, - db_conn: LoggingDatabaseConnection, - hs: "HomeServer", - ): - super().__init__(database, db_conn, hs) - - self._push_rule_id_gen = IdGenerator(db_conn, "push_rules", "id") - self._push_rules_enable_id_gen = IdGenerator(db_conn, "push_rules_enable", "id") - async def add_push_rule( self, user_id: str, @@ -410,6 +396,9 @@ class PushRuleStore(PushRulesWorkerStore): before: Optional[str] = None, after: Optional[str] = None, ) -> None: + if not self._is_push_writer: + raise Exception("Not a push writer") + conditions_json = json_encoder.encode(conditions) actions_json = json_encoder.encode(actions) async with self._push_rules_stream_id_gen.get_next() as stream_id: @@ -455,6 +444,9 @@ class PushRuleStore(PushRulesWorkerStore): before: str, after: str, ) -> None: + if not self._is_push_writer: + raise Exception("Not a push writer") + relative_to_rule = before or after sql = """ @@ -524,6 +516,9 @@ class PushRuleStore(PushRulesWorkerStore): conditions_json: str, actions_json: str, ) -> None: + if not self._is_push_writer: + raise Exception("Not a push writer") + if isinstance(self.database_engine, PostgresEngine): # Postgres doesn't do FOR UPDATE on aggregate functions, so select the rows first # then re-select the count/max below. @@ -575,6 +570,9 @@ class PushRuleStore(PushRulesWorkerStore): actions_json: str, update_stream: bool = True, ) -> None: + if not self._is_push_writer: + raise Exception("Not a push writer") + """Specialised version of simple_upsert_txn that picks a push_rule_id using the _push_rule_id_gen if it needs to insert the rule. It assumes that the "push_rules" table is locked""" @@ -653,6 +651,8 @@ class PushRuleStore(PushRulesWorkerStore): user_id: The matrix ID of the push rule owner rule_id: The rule_id of the rule to be deleted """ + if not self._is_push_writer: + raise Exception("Not a push writer") def delete_push_rule_txn( txn: LoggingTransaction, @@ -704,6 +704,9 @@ class PushRuleStore(PushRulesWorkerStore): Raises: RuleNotFoundException if the rule does not exist. """ + if not self._is_push_writer: + raise Exception("Not a push writer") + async with self._push_rules_stream_id_gen.get_next() as stream_id: event_stream_ordering = self._stream_id_gen.get_current_token() await self.db_pool.runInteraction( @@ -727,6 +730,9 @@ class PushRuleStore(PushRulesWorkerStore): enabled: bool, is_default_rule: bool, ) -> None: + if not self._is_push_writer: + raise Exception("Not a push writer") + new_id = self._push_rules_enable_id_gen.get_next() if not is_default_rule: @@ -796,6 +802,9 @@ class PushRuleStore(PushRulesWorkerStore): Raises: RuleNotFoundException if the rule does not exist. """ + if not self._is_push_writer: + raise Exception("Not a push writer") + actions_json = json_encoder.encode(actions) def set_push_rule_actions_txn( @@ -865,6 +874,9 @@ class PushRuleStore(PushRulesWorkerStore): op: str, data: Optional[JsonDict] = None, ) -> None: + if not self._is_push_writer: + raise Exception("Not a push writer") + values = { "stream_id": stream_id, "event_stream_ordering": event_stream_ordering, @@ -882,9 +894,6 @@ class PushRuleStore(PushRulesWorkerStore): self.push_rules_stream_cache.entity_has_changed, user_id, stream_id ) - def get_max_push_rules_stream_id(self) -> int: - return self._push_rules_stream_id_gen.get_current_token() - async def copy_push_rule_from_room_to_room( self, new_room_id: str, user_id: str, rule: PushRule ) -> None: @@ -895,6 +904,9 @@ class PushRuleStore(PushRulesWorkerStore): user_id : ID of user the push rule belongs to. rule: A push rule. """ + if not self._is_push_writer: + raise Exception("Not a push writer") + # Create new rule id rule_id_scope = "/".join(rule.rule_id.split("/")[:-1]) new_rule_id = rule_id_scope + "/" + new_room_id @@ -930,6 +942,9 @@ class PushRuleStore(PushRulesWorkerStore): new_room_id: ID of the new room. user_id: ID of user to copy push rules for. """ + if not self._is_push_writer: + raise Exception("Not a push writer") + # Retrieve push rules for this user user_push_rules = await self.get_push_rules_for_user(user_id) From fd48fc45853eb193a22a08d874eb473e668e2d6a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 28 Mar 2024 16:29:23 +0000 Subject: [PATCH 5/5] Fixups to new push stream (#17038) Follow on from #17037 --- changelog.d/17037.feature | 2 +- changelog.d/17038.feature | 1 + docker/configure_workers_and_start.py | 8 ++++++++ docs/workers.md | 4 ++-- synapse/config/workers.py | 8 ++++---- synapse/handlers/room_member.py | 6 ++++-- synapse/replication/tcp/handler.py | 2 +- synapse/rest/client/push_rule.py | 4 +++- synapse/storage/databases/main/push_rule.py | 4 +++- 9 files changed, 27 insertions(+), 12 deletions(-) create mode 100644 changelog.d/17038.feature diff --git a/changelog.d/17037.feature b/changelog.d/17037.feature index bd419c817d..498221e19e 100644 --- a/changelog.d/17037.feature +++ b/changelog.d/17037.feature @@ -1 +1 @@ -Add support for moving `/push_rules` off of main process. +Add support for moving `/pushrules` off of main process. diff --git a/changelog.d/17038.feature b/changelog.d/17038.feature new file mode 100644 index 0000000000..498221e19e --- /dev/null +++ b/changelog.d/17038.feature @@ -0,0 +1 @@ +Add support for moving `/pushrules` off of main process. diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 3917d9ae7e..77534a4f4f 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -310,6 +310,13 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "shared_extra_conf": {}, "worker_extra_conf": "", }, + "push_rules": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|v3|unstable)/pushrules/"], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, } # Templates for sections that may be inserted multiple times in config files @@ -401,6 +408,7 @@ def add_worker_roles_to_shared_config( "receipts", "to_device", "typing", + "push_rules", ] # Worker-type specific sharding config. Now a single worker can fulfill multiple diff --git a/docs/workers.md b/docs/workers.md index 5ea8ad59bd..ab9c1db86b 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -532,12 +532,12 @@ the stream writer for the `presence` stream: ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/ -##### The `push` stream +##### The `push_rules` stream The following endpoints should be routed directly to the worker configured as the stream writer for the `push` stream: - ^/_matrix/client/(api/v1|r0|v3|unstable)/push_rules/ + ^/_matrix/client/(api/v1|r0|v3|unstable)/pushrules/ #### Restrict outbound federation traffic to a specific set of workers diff --git a/synapse/config/workers.py b/synapse/config/workers.py index 9f81a73d6f..7ecf349e4a 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -156,7 +156,7 @@ class WriterLocations: can only be a single instance. presence: The instances that write to the presence stream. Currently can only be a single instance. - push: The instances that write to the push stream. Currently + push_rules: The instances that write to the push stream. Currently can only be a single instance. """ @@ -184,7 +184,7 @@ class WriterLocations: default=["master"], converter=_instance_to_list_converter, ) - push: List[str] = attr.ib( + push_rules: List[str] = attr.ib( default=["master"], converter=_instance_to_list_converter, ) @@ -347,7 +347,7 @@ class WorkerConfig(Config): "account_data", "receipts", "presence", - "push", + "push_rules", ): instances = _instance_to_list_converter(getattr(self.writers, stream)) for instance in instances: @@ -385,7 +385,7 @@ class WorkerConfig(Config): "Must only specify one instance to handle `presence` messages." ) - if len(self.writers.push) != 1: + if len(self.writers.push_rules) != 1: raise ConfigError( "Must only specify one instance to handle `push` messages." ) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index ee2e807afc..601d37341b 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -182,8 +182,10 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): hs.config.server.forgotten_room_retention_period ) - self._is_push_writer = hs.get_instance_name() in hs.config.worker.writers.push - self._push_writer = hs.config.worker.writers.push[0] + self._is_push_writer = ( + hs.get_instance_name() in hs.config.worker.writers.push_rules + ) + self._push_writer = hs.config.worker.writers.push_rules[0] self._copy_push_client = ReplicationCopyPusherRestServlet.make_client(hs) def _on_user_joined_room(self, event_id: str, room_id: str) -> None: diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index 4342d6ce70..72a42cb6cc 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -180,7 +180,7 @@ class ReplicationCommandHandler: continue if isinstance(stream, PushRulesStream): - if hs.get_instance_name() in hs.config.worker.writers.push: + if hs.get_instance_name() in hs.config.worker.writers.push_rules: self._streams_to_replicate.append(stream) continue diff --git a/synapse/rest/client/push_rule.py b/synapse/rest/client/push_rule.py index 9c9bfc9794..af042504c9 100644 --- a/synapse/rest/client/push_rule.py +++ b/synapse/rest/client/push_rule.py @@ -59,7 +59,9 @@ class PushRuleRestServlet(RestServlet): self.auth = hs.get_auth() self.store = hs.get_datastores().main self.notifier = hs.get_notifier() - self._is_push_worker = hs.get_instance_name() in hs.config.worker.writers.push + self._is_push_worker = ( + hs.get_instance_name() in hs.config.worker.writers.push_rules + ) self._push_rules_handler = hs.get_push_rules_handler() self._push_rule_linearizer = Linearizer(name="push_rules") diff --git a/synapse/storage/databases/main/push_rule.py b/synapse/storage/databases/main/push_rule.py index ed734f03ac..660c834518 100644 --- a/synapse/storage/databases/main/push_rule.py +++ b/synapse/storage/databases/main/push_rule.py @@ -136,7 +136,9 @@ class PushRulesWorkerStore( ): super().__init__(database, db_conn, hs) - self._is_push_writer = hs.get_instance_name() in hs.config.worker.writers.push + self._is_push_writer = ( + hs.get_instance_name() in hs.config.worker.writers.push_rules + ) # In the worker store this is an ID tracker which we overwrite in the non-worker # class below that is used on the main process.