From 4aa914369b966a76caa2724fbeee78b0336efdaa Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 27 Mar 2019 10:23:03 +0000 Subject: [PATCH 01/17] bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 25c10244d3..4a3a3ba236 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.99.2" +__version__ = "0.99.3rc1" From 6a69bf67db47ba723372d60295da86f0ebcbf062 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 27 Mar 2019 10:24:24 +0000 Subject: [PATCH 02/17] 0.99.3rc1 --- CHANGES.md | 93 ++++++++++++++++++++++++++++++++++++++++ changelog.d/2090.bugfix | 1 - changelog.d/4537.feature | 1 - changelog.d/4662.misc | 1 - changelog.d/4699.bugfix | 1 - changelog.d/4735.feature | 1 - changelog.d/4740.bugfix | 1 - changelog.d/4749.bugfix | 1 - changelog.d/4752.misc | 1 - changelog.d/4757.feature | 1 - changelog.d/4757.misc | 1 - changelog.d/4759.feature | 1 - changelog.d/4763.bugfix | 1 - changelog.d/4765.misc | 1 - changelog.d/4770.misc | 1 - changelog.d/4771.misc | 1 - changelog.d/4772.feature | 1 - changelog.d/4776.bugfix | 1 - changelog.d/4779.misc | 1 - changelog.d/4790.bugfix | 1 - changelog.d/4791.feature | 1 - changelog.d/4792.bugfix | 1 - changelog.d/4793.feature | 1 - changelog.d/4794.misc | 1 - changelog.d/4795.misc | 1 - changelog.d/4796.feature | 1 - changelog.d/4797.misc | 1 - changelog.d/4798.misc | 1 - changelog.d/4799.misc | 1 - changelog.d/4801.feature | 1 - changelog.d/4804.feature | 1 - changelog.d/4814.feature | 1 - changelog.d/4815.misc | 1 - changelog.d/4816.misc | 1 - changelog.d/4817.misc | 1 - changelog.d/4818.bugfix | 1 - changelog.d/4820.misc | 1 - changelog.d/4821.feature | 1 - changelog.d/4824.misc | 1 - changelog.d/4825.misc | 1 - changelog.d/4828.misc | 1 - changelog.d/4829.bugfix | 1 - changelog.d/4832.misc | 1 - changelog.d/4837.bugfix | 1 - changelog.d/4838.bugfix | 1 - changelog.d/4839.misc | 1 - changelog.d/4840.feature | 1 - changelog.d/4843.misc | 1 - changelog.d/4844.misc | 1 - changelog.d/4846.feature | 1 - changelog.d/4847.misc | 1 - changelog.d/4849.misc | 1 - changelog.d/4852.misc | 1 - changelog.d/4853.feature | 1 - changelog.d/4855.misc | 1 - changelog.d/4863.misc | 1 - changelog.d/4864.feature | 1 - changelog.d/4865.feature | 1 - changelog.d/4869.misc | 1 - changelog.d/4870.misc | 1 - changelog.d/4879.misc | 1 - changelog.d/4881.misc | 1 - changelog.d/4886.bugfix | 1 - changelog.d/4886.misc | 1 - changelog.d/4887.feature | 1 - changelog.d/4888.bugfix | 2 - changelog.d/4889.misc | 1 - changelog.d/4890.feature | 1 - changelog.d/4895.feature | 1 - changelog.d/4895.misc | 1 - changelog.d/4896.feature | 1 - changelog.d/4900.feature | 1 - changelog.d/4902.misc | 1 - changelog.d/4904.bugfix | 1 - changelog.d/4905.misc | 1 - changelog.d/4908.bugfix | 1 - changelog.d/4912.misc | 1 - changelog.d/4913.misc | 1 - changelog.d/4917.misc | 1 - changelog.d/4923.misc | 1 - changelog.d/4927.feature | 1 - changelog.d/4928.misc | 1 - changelog.d/4929.misc | 1 - changelog.d/4931.feature | 1 - changelog.d/4944.feature | 1 - 85 files changed, 93 insertions(+), 85 deletions(-) delete mode 100644 changelog.d/2090.bugfix delete mode 100644 changelog.d/4537.feature delete mode 100644 changelog.d/4662.misc delete mode 100644 changelog.d/4699.bugfix delete mode 100644 changelog.d/4735.feature delete mode 100644 changelog.d/4740.bugfix delete mode 100644 changelog.d/4749.bugfix delete mode 100644 changelog.d/4752.misc delete mode 100644 changelog.d/4757.feature delete mode 100644 changelog.d/4757.misc delete mode 100644 changelog.d/4759.feature delete mode 100644 changelog.d/4763.bugfix delete mode 100644 changelog.d/4765.misc delete mode 100644 changelog.d/4770.misc delete mode 100644 changelog.d/4771.misc delete mode 100644 changelog.d/4772.feature delete mode 100644 changelog.d/4776.bugfix delete mode 100644 changelog.d/4779.misc delete mode 100644 changelog.d/4790.bugfix delete mode 100644 changelog.d/4791.feature delete mode 100644 changelog.d/4792.bugfix delete mode 100644 changelog.d/4793.feature delete mode 100644 changelog.d/4794.misc delete mode 100644 changelog.d/4795.misc delete mode 100644 changelog.d/4796.feature delete mode 100644 changelog.d/4797.misc delete mode 100644 changelog.d/4798.misc delete mode 100644 changelog.d/4799.misc delete mode 100644 changelog.d/4801.feature delete mode 100644 changelog.d/4804.feature delete mode 100644 changelog.d/4814.feature delete mode 100644 changelog.d/4815.misc delete mode 100644 changelog.d/4816.misc delete mode 100644 changelog.d/4817.misc delete mode 100644 changelog.d/4818.bugfix delete mode 100644 changelog.d/4820.misc delete mode 100644 changelog.d/4821.feature delete mode 100644 changelog.d/4824.misc delete mode 100644 changelog.d/4825.misc delete mode 100644 changelog.d/4828.misc delete mode 100644 changelog.d/4829.bugfix delete mode 100644 changelog.d/4832.misc delete mode 100644 changelog.d/4837.bugfix delete mode 100644 changelog.d/4838.bugfix delete mode 100644 changelog.d/4839.misc delete mode 100644 changelog.d/4840.feature delete mode 100644 changelog.d/4843.misc delete mode 100644 changelog.d/4844.misc delete mode 100644 changelog.d/4846.feature delete mode 100644 changelog.d/4847.misc delete mode 100644 changelog.d/4849.misc delete mode 100644 changelog.d/4852.misc delete mode 100644 changelog.d/4853.feature delete mode 100644 changelog.d/4855.misc delete mode 100644 changelog.d/4863.misc delete mode 100644 changelog.d/4864.feature delete mode 100644 changelog.d/4865.feature delete mode 100644 changelog.d/4869.misc delete mode 100644 changelog.d/4870.misc delete mode 100644 changelog.d/4879.misc delete mode 100644 changelog.d/4881.misc delete mode 100644 changelog.d/4886.bugfix delete mode 100644 changelog.d/4886.misc delete mode 100644 changelog.d/4887.feature delete mode 100644 changelog.d/4888.bugfix delete mode 100644 changelog.d/4889.misc delete mode 100644 changelog.d/4890.feature delete mode 100644 changelog.d/4895.feature delete mode 100644 changelog.d/4895.misc delete mode 100644 changelog.d/4896.feature delete mode 100644 changelog.d/4900.feature delete mode 100644 changelog.d/4902.misc delete mode 100644 changelog.d/4904.bugfix delete mode 100644 changelog.d/4905.misc delete mode 100644 changelog.d/4908.bugfix delete mode 100644 changelog.d/4912.misc delete mode 100644 changelog.d/4913.misc delete mode 100644 changelog.d/4917.misc delete mode 100644 changelog.d/4923.misc delete mode 100644 changelog.d/4927.feature delete mode 100644 changelog.d/4928.misc delete mode 100644 changelog.d/4929.misc delete mode 100644 changelog.d/4931.feature delete mode 100644 changelog.d/4944.feature diff --git a/CHANGES.md b/CHANGES.md index b25775d18e..b13a324037 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,96 @@ +Synapse 0.99.3rc1 (2019-03-27) +============================== + +Features +-------- + +- The user directory has been rewritten to make it faster, with less chance of falling behind on a large server. ([\#4537](https://github.com/matrix-org/synapse/issues/4537), [\#4846](https://github.com/matrix-org/synapse/issues/4846), [\#4864](https://github.com/matrix-org/synapse/issues/4864), [\#4887](https://github.com/matrix-org/synapse/issues/4887), [\#4900](https://github.com/matrix-org/synapse/issues/4900), [\#4944](https://github.com/matrix-org/synapse/issues/4944)) +- Add configurable rate limiting to the /register endpoint. ([\#4735](https://github.com/matrix-org/synapse/issues/4735), [\#4804](https://github.com/matrix-org/synapse/issues/4804)) +- Move server key queries to federation reader. ([\#4757](https://github.com/matrix-org/synapse/issues/4757)) +- Add support for /account/3pid REST endpoint to client_reader worker. ([\#4759](https://github.com/matrix-org/synapse/issues/4759)) +- Add an endpoint to the admin API for querying the server version. Contributed by Joseph Weston. ([\#4772](https://github.com/matrix-org/synapse/issues/4772)) +- Include a default configuration file in the 'docs' directory. ([\#4791](https://github.com/matrix-org/synapse/issues/4791), [\#4801](https://github.com/matrix-org/synapse/issues/4801)) +- Synapse is now permissive about trailing slashes on some of its federation endpoints, allowing zero or more to be present. ([\#4793](https://github.com/matrix-org/synapse/issues/4793)) +- Add support for /keys/query and /keys/changes REST endpoints to client_reader worker. ([\#4796](https://github.com/matrix-org/synapse/issues/4796)) +- Add checks to incoming events over federation for events evading auth (aka "soft fail"). ([\#4814](https://github.com/matrix-org/synapse/issues/4814)) +- Add configurable rate limiting to the /login endpoint. ([\#4821](https://github.com/matrix-org/synapse/issues/4821), [\#4865](https://github.com/matrix-org/synapse/issues/4865)) +- Remove trailing slashes from certain outbound federation requests. Retry if receiving a 404. Context: #3622. ([\#4840](https://github.com/matrix-org/synapse/issues/4840)) +- Allow passing --daemonize flags to workers in the same way as with master. ([\#4853](https://github.com/matrix-org/synapse/issues/4853)) +- Batch up outgoing read-receipts to reduce federation traffic. ([\#4890](https://github.com/matrix-org/synapse/issues/4890), [\#4927](https://github.com/matrix-org/synapse/issues/4927)) +- Add option to disable searching the user directory. ([\#4895](https://github.com/matrix-org/synapse/issues/4895)) +- Add option to disable searching of local and remote public room lists. ([\#4896](https://github.com/matrix-org/synapse/issues/4896)) +- Add ability for password providers to login/register a user via 3PID (email, phone). ([\#4931](https://github.com/matrix-org/synapse/issues/4931)) + + +Bugfixes +-------- + +- Fix a bug where media with spaces in the name would get a corrupted name. ([\#2090](https://github.com/matrix-org/synapse/issues/2090)) +- Fix attempting to paginate in rooms where server cannot see any events, to avoid unnecessarily pulling in lots of redacted events. ([\#4699](https://github.com/matrix-org/synapse/issues/4699)) +- 'event_id' is now a required parameter in federated state requests, as per the matrix spec. ([\#4740](https://github.com/matrix-org/synapse/issues/4740)) +- Fix tightloop over connecting to replication server. ([\#4749](https://github.com/matrix-org/synapse/issues/4749)) +- Fix parsing of Content-Disposition headers on remote media requests and URL previews. ([\#4763](https://github.com/matrix-org/synapse/issues/4763)) +- Fix incorrect log about not persisting duplicate state event. ([\#4776](https://github.com/matrix-org/synapse/issues/4776)) +- Fix v4v6 option in HAProxy example config. Contributed by Flakebi. ([\#4790](https://github.com/matrix-org/synapse/issues/4790)) +- Handle batch updates in worker replication protocol. ([\#4792](https://github.com/matrix-org/synapse/issues/4792)) +- Fix bug where we didn't correctly throttle sending of USER_IP commands over replication. ([\#4818](https://github.com/matrix-org/synapse/issues/4818)) +- Fix potential race in handling missing updates in device list updates. ([\#4829](https://github.com/matrix-org/synapse/issues/4829)) +- Fix bug where synapse expected an un-specced `prev_state` field on state events. ([\#4837](https://github.com/matrix-org/synapse/issues/4837)) +- Transfer a user's notification settings (push rules) on room upgrade. ([\#4838](https://github.com/matrix-org/synapse/issues/4838)) +- fix test_auto_create_auto_join_where_no_consent. ([\#4886](https://github.com/matrix-org/synapse/issues/4886)) +- Fix a bug where hs_disabled_message was sometimes not correctly enforced. ([\#4888](https://github.com/matrix-org/synapse/issues/4888)) +- Fix bug in shutdown room admin API where it would fail if a user in the room hadn't consented to the privacy policy. ([\#4904](https://github.com/matrix-org/synapse/issues/4904)) +- Fix bug where blocked world-readable rooms were still peekable. ([\#4908](https://github.com/matrix-org/synapse/issues/4908)) + + +Internal Changes +---------------- + +- Add a systemd setup that supports synapse workers. Contributed by Luca Corbatto. ([\#4662](https://github.com/matrix-org/synapse/issues/4662)) +- Change from TravisCI to Buildkite for CI. ([\#4752](https://github.com/matrix-org/synapse/issues/4752)) +- When presence is disabled don't send over replication. ([\#4757](https://github.com/matrix-org/synapse/issues/4757)) +- Minor docstring fixes for MatrixFederationAgent. ([\#4765](https://github.com/matrix-org/synapse/issues/4765)) +- Optimise EDU transmission for the federation_sender worker. ([\#4770](https://github.com/matrix-org/synapse/issues/4770)) +- Update test_typing to use HomeserverTestCase. ([\#4771](https://github.com/matrix-org/synapse/issues/4771)) +- Update URLs for riot.im icons and logos in the default notification templates. ([\#4779](https://github.com/matrix-org/synapse/issues/4779)) +- Removed unnecessary $ from some federation endpoint path regexes. ([\#4794](https://github.com/matrix-org/synapse/issues/4794)) +- Remove link to deleted title in README. ([\#4795](https://github.com/matrix-org/synapse/issues/4795)) +- Clean up read-receipt handling. ([\#4797](https://github.com/matrix-org/synapse/issues/4797)) +- Add some debug about processing read receipts. ([\#4798](https://github.com/matrix-org/synapse/issues/4798)) +- Clean up some replication code. ([\#4799](https://github.com/matrix-org/synapse/issues/4799)) +- Add some docstrings. ([\#4815](https://github.com/matrix-org/synapse/issues/4815)) +- Add debug logger to try and track down #4422. ([\#4816](https://github.com/matrix-org/synapse/issues/4816)) +- Make shutdown API send explanation message to room after users have been forced joined. ([\#4817](https://github.com/matrix-org/synapse/issues/4817)) +- Update example_log_config.yaml. ([\#4820](https://github.com/matrix-org/synapse/issues/4820)) +- Document the `generate` option for the docker image. ([\#4824](https://github.com/matrix-org/synapse/issues/4824)) +- Fix check-newsfragment for debian-only changes. ([\#4825](https://github.com/matrix-org/synapse/issues/4825)) +- Add some debug logging for device list updates to help with #4828. ([\#4828](https://github.com/matrix-org/synapse/issues/4828)) +- Improve federation documentation, specifically .well-known support. Many thanks to @vaab. ([\#4832](https://github.com/matrix-org/synapse/issues/4832)) +- Disable captcha registration by default in unit tests. ([\#4839](https://github.com/matrix-org/synapse/issues/4839)) +- Add stuff back to the .gitignore. ([\#4843](https://github.com/matrix-org/synapse/issues/4843)) +- Clarify what registration_shared_secret allows for. ([\#4844](https://github.com/matrix-org/synapse/issues/4844)) +- Correctly log expected errors when fetching server keys. ([\#4847](https://github.com/matrix-org/synapse/issues/4847)) +- Update install docs to explicitly state a full-chain (not just the top-level) TLS certificate must be provided to Synapse. This caused some people's Synapse ports to appear correct in a browser but still (rightfully so) upset the federation tester. ([\#4849](https://github.com/matrix-org/synapse/issues/4849)) +- Move client read-receipt processing to federation sender worker. ([\#4852](https://github.com/matrix-org/synapse/issues/4852)) +- Refactor federation TransactionQueue. ([\#4855](https://github.com/matrix-org/synapse/issues/4855)) +- Comment out most options in the generated config. ([\#4863](https://github.com/matrix-org/synapse/issues/4863)) +- Fix yaml library warnings by using safe_load. ([\#4869](https://github.com/matrix-org/synapse/issues/4869)) +- Update Apache setup to remove location syntax. Thanks to @cwmke! ([\#4870](https://github.com/matrix-org/synapse/issues/4870)) +- Reinstate test case that runs unit tests against oldest supported dependencies. ([\#4879](https://github.com/matrix-org/synapse/issues/4879)) +- Update link to federation docs. ([\#4881](https://github.com/matrix-org/synapse/issues/4881)) +- fix test_auto_create_auto_join_where_no_consent. ([\#4886](https://github.com/matrix-org/synapse/issues/4886)) +- Use a regular HomeServerConfig object for unit tests rater than a Mock. ([\#4889](https://github.com/matrix-org/synapse/issues/4889)) +- Add some notes about tuning postgres for larger deployments. ([\#4895](https://github.com/matrix-org/synapse/issues/4895)) +- Add a config option for torture-testing worker replication. ([\#4902](https://github.com/matrix-org/synapse/issues/4902)) +- Log requests which are simulated by the unit tests. ([\#4905](https://github.com/matrix-org/synapse/issues/4905)) +- Allow newsfragments to end with exclamation marks. Exciting! ([\#4912](https://github.com/matrix-org/synapse/issues/4912)) +- Refactor some more tests to use HomeserverTestCase. ([\#4913](https://github.com/matrix-org/synapse/issues/4913)) +- Refactor out the state deltas portion of the user directory store and handler. ([\#4917](https://github.com/matrix-org/synapse/issues/4917)) +- Fix nginx example in ACME doc. ([\#4923](https://github.com/matrix-org/synapse/issues/4923)) +- Use an explicit dbname for postgres connections in the tests. ([\#4928](https://github.com/matrix-org/synapse/issues/4928)) +- Fix `ClientReplicationStreamProtocol.__str__()`. ([\#4929](https://github.com/matrix-org/synapse/issues/4929)) + + Synapse 0.99.2 (2019-03-01) =========================== diff --git a/changelog.d/2090.bugfix b/changelog.d/2090.bugfix deleted file mode 100644 index de2d22fcb8..0000000000 --- a/changelog.d/2090.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where media with spaces in the name would get a corrupted name. diff --git a/changelog.d/4537.feature b/changelog.d/4537.feature deleted file mode 100644 index 8f792b8890..0000000000 --- a/changelog.d/4537.feature +++ /dev/null @@ -1 +0,0 @@ -The user directory has been rewritten to make it faster, with less chance of falling behind on a large server. diff --git a/changelog.d/4662.misc b/changelog.d/4662.misc deleted file mode 100644 index f4ec0d6a68..0000000000 --- a/changelog.d/4662.misc +++ /dev/null @@ -1 +0,0 @@ -Add a systemd setup that supports synapse workers. Contributed by Luca Corbatto. diff --git a/changelog.d/4699.bugfix b/changelog.d/4699.bugfix deleted file mode 100644 index 1d7f3174e7..0000000000 --- a/changelog.d/4699.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix attempting to paginate in rooms where server cannot see any events, to avoid unnecessarily pulling in lots of redacted events. diff --git a/changelog.d/4735.feature b/changelog.d/4735.feature deleted file mode 100644 index a4c0b196f6..0000000000 --- a/changelog.d/4735.feature +++ /dev/null @@ -1 +0,0 @@ -Add configurable rate limiting to the /register endpoint. diff --git a/changelog.d/4740.bugfix b/changelog.d/4740.bugfix deleted file mode 100644 index f82bb4227a..0000000000 --- a/changelog.d/4740.bugfix +++ /dev/null @@ -1 +0,0 @@ -'event_id' is now a required parameter in federated state requests, as per the matrix spec. diff --git a/changelog.d/4749.bugfix b/changelog.d/4749.bugfix deleted file mode 100644 index 174e6b4e5e..0000000000 --- a/changelog.d/4749.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix tightloop over connecting to replication server. diff --git a/changelog.d/4752.misc b/changelog.d/4752.misc deleted file mode 100644 index fb1e76edce..0000000000 --- a/changelog.d/4752.misc +++ /dev/null @@ -1 +0,0 @@ -Change from TravisCI to Buildkite for CI. diff --git a/changelog.d/4757.feature b/changelog.d/4757.feature deleted file mode 100644 index b89029f2b4..0000000000 --- a/changelog.d/4757.feature +++ /dev/null @@ -1 +0,0 @@ -Move server key queries to federation reader. diff --git a/changelog.d/4757.misc b/changelog.d/4757.misc deleted file mode 100644 index 42bb66f7aa..0000000000 --- a/changelog.d/4757.misc +++ /dev/null @@ -1 +0,0 @@ -When presence is disabled don't send over replication. diff --git a/changelog.d/4759.feature b/changelog.d/4759.feature deleted file mode 100644 index 643ee404dc..0000000000 --- a/changelog.d/4759.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for /account/3pid REST endpoint to client_reader worker. diff --git a/changelog.d/4763.bugfix b/changelog.d/4763.bugfix deleted file mode 100644 index 213ea44b70..0000000000 --- a/changelog.d/4763.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix parsing of Content-Disposition headers on remote media requests and URL previews. diff --git a/changelog.d/4765.misc b/changelog.d/4765.misc deleted file mode 100644 index c273fd0cc4..0000000000 --- a/changelog.d/4765.misc +++ /dev/null @@ -1 +0,0 @@ -Minor docstring fixes for MatrixFederationAgent. \ No newline at end of file diff --git a/changelog.d/4770.misc b/changelog.d/4770.misc deleted file mode 100644 index 144d819958..0000000000 --- a/changelog.d/4770.misc +++ /dev/null @@ -1 +0,0 @@ -Optimise EDU transmission for the federation_sender worker. diff --git a/changelog.d/4771.misc b/changelog.d/4771.misc deleted file mode 100644 index 8fa3401ca4..0000000000 --- a/changelog.d/4771.misc +++ /dev/null @@ -1 +0,0 @@ -Update test_typing to use HomeserverTestCase. diff --git a/changelog.d/4772.feature b/changelog.d/4772.feature deleted file mode 100644 index 19bb91f1e8..0000000000 --- a/changelog.d/4772.feature +++ /dev/null @@ -1 +0,0 @@ -Add an endpoint to the admin API for querying the server version. Contributed by Joseph Weston. diff --git a/changelog.d/4776.bugfix b/changelog.d/4776.bugfix deleted file mode 100644 index ce3e6ce33c..0000000000 --- a/changelog.d/4776.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix incorrect log about not persisting duplicate state event. diff --git a/changelog.d/4779.misc b/changelog.d/4779.misc deleted file mode 100644 index 2442bf31bd..0000000000 --- a/changelog.d/4779.misc +++ /dev/null @@ -1 +0,0 @@ -Update URLs for riot.im icons and logos in the default notification templates. diff --git a/changelog.d/4790.bugfix b/changelog.d/4790.bugfix deleted file mode 100644 index aa8eb93246..0000000000 --- a/changelog.d/4790.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix v4v6 option in HAProxy example config. Contributed by Flakebi. diff --git a/changelog.d/4791.feature b/changelog.d/4791.feature deleted file mode 100644 index 1e5fd32463..0000000000 --- a/changelog.d/4791.feature +++ /dev/null @@ -1 +0,0 @@ -Include a default configuration file in the 'docs' directory. diff --git a/changelog.d/4792.bugfix b/changelog.d/4792.bugfix deleted file mode 100644 index b127b6254f..0000000000 --- a/changelog.d/4792.bugfix +++ /dev/null @@ -1 +0,0 @@ -Handle batch updates in worker replication protocol. \ No newline at end of file diff --git a/changelog.d/4793.feature b/changelog.d/4793.feature deleted file mode 100644 index 90dba7d122..0000000000 --- a/changelog.d/4793.feature +++ /dev/null @@ -1 +0,0 @@ -Synapse is now permissive about trailing slashes on some of its federation endpoints, allowing zero or more to be present. \ No newline at end of file diff --git a/changelog.d/4794.misc b/changelog.d/4794.misc deleted file mode 100644 index 99b543ecba..0000000000 --- a/changelog.d/4794.misc +++ /dev/null @@ -1 +0,0 @@ -Removed unnecessary $ from some federation endpoint path regexes. \ No newline at end of file diff --git a/changelog.d/4795.misc b/changelog.d/4795.misc deleted file mode 100644 index 03995f42fe..0000000000 --- a/changelog.d/4795.misc +++ /dev/null @@ -1 +0,0 @@ -Remove link to deleted title in README. \ No newline at end of file diff --git a/changelog.d/4796.feature b/changelog.d/4796.feature deleted file mode 100644 index 9e05560a3f..0000000000 --- a/changelog.d/4796.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for /keys/query and /keys/changes REST endpoints to client_reader worker. diff --git a/changelog.d/4797.misc b/changelog.d/4797.misc deleted file mode 100644 index 822e98e6a7..0000000000 --- a/changelog.d/4797.misc +++ /dev/null @@ -1 +0,0 @@ -Clean up read-receipt handling. diff --git a/changelog.d/4798.misc b/changelog.d/4798.misc deleted file mode 100644 index d60f208dc3..0000000000 --- a/changelog.d/4798.misc +++ /dev/null @@ -1 +0,0 @@ -Add some debug about processing read receipts. diff --git a/changelog.d/4799.misc b/changelog.d/4799.misc deleted file mode 100644 index 5ab11a5c0b..0000000000 --- a/changelog.d/4799.misc +++ /dev/null @@ -1 +0,0 @@ -Clean up some replication code. diff --git a/changelog.d/4801.feature b/changelog.d/4801.feature deleted file mode 100644 index 1e5fd32463..0000000000 --- a/changelog.d/4801.feature +++ /dev/null @@ -1 +0,0 @@ -Include a default configuration file in the 'docs' directory. diff --git a/changelog.d/4804.feature b/changelog.d/4804.feature deleted file mode 100644 index a4c0b196f6..0000000000 --- a/changelog.d/4804.feature +++ /dev/null @@ -1 +0,0 @@ -Add configurable rate limiting to the /register endpoint. diff --git a/changelog.d/4814.feature b/changelog.d/4814.feature deleted file mode 100644 index 9433acd959..0000000000 --- a/changelog.d/4814.feature +++ /dev/null @@ -1 +0,0 @@ -Add checks to incoming events over federation for events evading auth (aka "soft fail"). diff --git a/changelog.d/4815.misc b/changelog.d/4815.misc deleted file mode 100644 index b123b36f7f..0000000000 --- a/changelog.d/4815.misc +++ /dev/null @@ -1 +0,0 @@ -Add some docstrings. diff --git a/changelog.d/4816.misc b/changelog.d/4816.misc deleted file mode 100644 index 43d94251f7..0000000000 --- a/changelog.d/4816.misc +++ /dev/null @@ -1 +0,0 @@ -Add debug logger to try and track down #4422. diff --git a/changelog.d/4817.misc b/changelog.d/4817.misc deleted file mode 100644 index 438a51dc63..0000000000 --- a/changelog.d/4817.misc +++ /dev/null @@ -1 +0,0 @@ -Make shutdown API send explanation message to room after users have been forced joined. diff --git a/changelog.d/4818.bugfix b/changelog.d/4818.bugfix deleted file mode 100644 index ebbc27a433..0000000000 --- a/changelog.d/4818.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where we didn't correctly throttle sending of USER_IP commands over replication. diff --git a/changelog.d/4820.misc b/changelog.d/4820.misc deleted file mode 100644 index 1e35b5b63c..0000000000 --- a/changelog.d/4820.misc +++ /dev/null @@ -1 +0,0 @@ -Update example_log_config.yaml. diff --git a/changelog.d/4821.feature b/changelog.d/4821.feature deleted file mode 100644 index 61d4eb8d60..0000000000 --- a/changelog.d/4821.feature +++ /dev/null @@ -1 +0,0 @@ -Add configurable rate limiting to the /login endpoint. diff --git a/changelog.d/4824.misc b/changelog.d/4824.misc deleted file mode 100644 index a4c5a1df37..0000000000 --- a/changelog.d/4824.misc +++ /dev/null @@ -1 +0,0 @@ -Document the `generate` option for the docker image. diff --git a/changelog.d/4825.misc b/changelog.d/4825.misc deleted file mode 100644 index 166661ab6a..0000000000 --- a/changelog.d/4825.misc +++ /dev/null @@ -1 +0,0 @@ -Fix check-newsfragment for debian-only changes. diff --git a/changelog.d/4828.misc b/changelog.d/4828.misc deleted file mode 100644 index 2fe554884a..0000000000 --- a/changelog.d/4828.misc +++ /dev/null @@ -1 +0,0 @@ -Add some debug logging for device list updates to help with #4828. diff --git a/changelog.d/4829.bugfix b/changelog.d/4829.bugfix deleted file mode 100644 index b05235e215..0000000000 --- a/changelog.d/4829.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix potential race in handling missing updates in device list updates. diff --git a/changelog.d/4832.misc b/changelog.d/4832.misc deleted file mode 100644 index 92022266c6..0000000000 --- a/changelog.d/4832.misc +++ /dev/null @@ -1 +0,0 @@ -Improve federation documentation, specifically .well-known support. Many thanks to @vaab. diff --git a/changelog.d/4837.bugfix b/changelog.d/4837.bugfix deleted file mode 100644 index 989aeb82bb..0000000000 --- a/changelog.d/4837.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where synapse expected an un-specced `prev_state` field on state events. diff --git a/changelog.d/4838.bugfix b/changelog.d/4838.bugfix deleted file mode 100644 index 7f4fceabff..0000000000 --- a/changelog.d/4838.bugfix +++ /dev/null @@ -1 +0,0 @@ -Transfer a user's notification settings (push rules) on room upgrade. \ No newline at end of file diff --git a/changelog.d/4839.misc b/changelog.d/4839.misc deleted file mode 100644 index 7c6868051b..0000000000 --- a/changelog.d/4839.misc +++ /dev/null @@ -1 +0,0 @@ -Disable captcha registration by default in unit tests. \ No newline at end of file diff --git a/changelog.d/4840.feature b/changelog.d/4840.feature deleted file mode 100644 index 9d1fd59053..0000000000 --- a/changelog.d/4840.feature +++ /dev/null @@ -1 +0,0 @@ -Remove trailing slashes from certain outbound federation requests. Retry if receiving a 404. Context: #3622. \ No newline at end of file diff --git a/changelog.d/4843.misc b/changelog.d/4843.misc deleted file mode 100644 index 03d0a3e2e7..0000000000 --- a/changelog.d/4843.misc +++ /dev/null @@ -1 +0,0 @@ -Add stuff back to the .gitignore. diff --git a/changelog.d/4844.misc b/changelog.d/4844.misc deleted file mode 100644 index eff6f1c43c..0000000000 --- a/changelog.d/4844.misc +++ /dev/null @@ -1 +0,0 @@ -Clarify what registration_shared_secret allows for. diff --git a/changelog.d/4846.feature b/changelog.d/4846.feature deleted file mode 100644 index 8f792b8890..0000000000 --- a/changelog.d/4846.feature +++ /dev/null @@ -1 +0,0 @@ -The user directory has been rewritten to make it faster, with less chance of falling behind on a large server. diff --git a/changelog.d/4847.misc b/changelog.d/4847.misc deleted file mode 100644 index a001238e08..0000000000 --- a/changelog.d/4847.misc +++ /dev/null @@ -1 +0,0 @@ -Correctly log expected errors when fetching server keys. diff --git a/changelog.d/4849.misc b/changelog.d/4849.misc deleted file mode 100644 index f2cab20b44..0000000000 --- a/changelog.d/4849.misc +++ /dev/null @@ -1 +0,0 @@ -Update install docs to explicitly state a full-chain (not just the top-level) TLS certificate must be provided to Synapse. This caused some people's Synapse ports to appear correct in a browser but still (rightfully so) upset the federation tester. \ No newline at end of file diff --git a/changelog.d/4852.misc b/changelog.d/4852.misc deleted file mode 100644 index 76ab1e34e7..0000000000 --- a/changelog.d/4852.misc +++ /dev/null @@ -1 +0,0 @@ - Move client read-receipt processing to federation sender worker. \ No newline at end of file diff --git a/changelog.d/4853.feature b/changelog.d/4853.feature deleted file mode 100644 index 360f92e1de..0000000000 --- a/changelog.d/4853.feature +++ /dev/null @@ -1 +0,0 @@ -Allow passing --daemonize flags to workers in the same way as with master. diff --git a/changelog.d/4855.misc b/changelog.d/4855.misc deleted file mode 100644 index c4906d2f56..0000000000 --- a/changelog.d/4855.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor federation TransactionQueue. \ No newline at end of file diff --git a/changelog.d/4863.misc b/changelog.d/4863.misc deleted file mode 100644 index bfe03cbedc..0000000000 --- a/changelog.d/4863.misc +++ /dev/null @@ -1 +0,0 @@ -Comment out most options in the generated config. diff --git a/changelog.d/4864.feature b/changelog.d/4864.feature deleted file mode 100644 index 57927f2620..0000000000 --- a/changelog.d/4864.feature +++ /dev/null @@ -1 +0,0 @@ -The user directory has been rewritten to make it faster, with less chance of falling behind on a large server. \ No newline at end of file diff --git a/changelog.d/4865.feature b/changelog.d/4865.feature deleted file mode 100644 index 61d4eb8d60..0000000000 --- a/changelog.d/4865.feature +++ /dev/null @@ -1 +0,0 @@ -Add configurable rate limiting to the /login endpoint. diff --git a/changelog.d/4869.misc b/changelog.d/4869.misc deleted file mode 100644 index d8186cc520..0000000000 --- a/changelog.d/4869.misc +++ /dev/null @@ -1 +0,0 @@ -Fix yaml library warnings by using safe_load. diff --git a/changelog.d/4870.misc b/changelog.d/4870.misc deleted file mode 100644 index f287b7d3b0..0000000000 --- a/changelog.d/4870.misc +++ /dev/null @@ -1 +0,0 @@ -Update Apache setup to remove location syntax. Thanks to @cwmke! diff --git a/changelog.d/4879.misc b/changelog.d/4879.misc deleted file mode 100644 index 574017230c..0000000000 --- a/changelog.d/4879.misc +++ /dev/null @@ -1 +0,0 @@ -Reinstate test case that runs unit tests against oldest supported dependencies. diff --git a/changelog.d/4881.misc b/changelog.d/4881.misc deleted file mode 100644 index 308c21c839..0000000000 --- a/changelog.d/4881.misc +++ /dev/null @@ -1 +0,0 @@ -Update link to federation docs. diff --git a/changelog.d/4886.bugfix b/changelog.d/4886.bugfix deleted file mode 100644 index b17aa92485..0000000000 --- a/changelog.d/4886.bugfix +++ /dev/null @@ -1 +0,0 @@ -fix test_auto_create_auto_join_where_no_consent. diff --git a/changelog.d/4886.misc b/changelog.d/4886.misc deleted file mode 100644 index b17aa92485..0000000000 --- a/changelog.d/4886.misc +++ /dev/null @@ -1 +0,0 @@ -fix test_auto_create_auto_join_where_no_consent. diff --git a/changelog.d/4887.feature b/changelog.d/4887.feature deleted file mode 100644 index e7ff0b9297..0000000000 --- a/changelog.d/4887.feature +++ /dev/null @@ -1 +0,0 @@ -The user directory has been rewritten to make it faster, with less chance of falling behind on a large server. diff --git a/changelog.d/4888.bugfix b/changelog.d/4888.bugfix deleted file mode 100644 index 0e193709e5..0000000000 --- a/changelog.d/4888.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug where hs_disabled_message was sometimes not correctly enforced. - diff --git a/changelog.d/4889.misc b/changelog.d/4889.misc deleted file mode 100644 index f1948db65e..0000000000 --- a/changelog.d/4889.misc +++ /dev/null @@ -1 +0,0 @@ -Use a regular HomeServerConfig object for unit tests rater than a Mock. diff --git a/changelog.d/4890.feature b/changelog.d/4890.feature deleted file mode 100644 index 8d74262250..0000000000 --- a/changelog.d/4890.feature +++ /dev/null @@ -1 +0,0 @@ -Batch up outgoing read-receipts to reduce federation traffic. diff --git a/changelog.d/4895.feature b/changelog.d/4895.feature deleted file mode 100644 index 5dd7c68194..0000000000 --- a/changelog.d/4895.feature +++ /dev/null @@ -1 +0,0 @@ -Add option to disable searching the user directory. diff --git a/changelog.d/4895.misc b/changelog.d/4895.misc deleted file mode 100644 index 81a3261538..0000000000 --- a/changelog.d/4895.misc +++ /dev/null @@ -1 +0,0 @@ -Add some notes about tuning postgres for larger deployments. diff --git a/changelog.d/4896.feature b/changelog.d/4896.feature deleted file mode 100644 index 46ac49a4b4..0000000000 --- a/changelog.d/4896.feature +++ /dev/null @@ -1 +0,0 @@ -Add option to disable searching of local and remote public room lists. diff --git a/changelog.d/4900.feature b/changelog.d/4900.feature deleted file mode 100644 index 8f792b8890..0000000000 --- a/changelog.d/4900.feature +++ /dev/null @@ -1 +0,0 @@ -The user directory has been rewritten to make it faster, with less chance of falling behind on a large server. diff --git a/changelog.d/4902.misc b/changelog.d/4902.misc deleted file mode 100644 index fecc06a6e8..0000000000 --- a/changelog.d/4902.misc +++ /dev/null @@ -1 +0,0 @@ -Add a config option for torture-testing worker replication. diff --git a/changelog.d/4904.bugfix b/changelog.d/4904.bugfix deleted file mode 100644 index 5c2d7f3cf1..0000000000 --- a/changelog.d/4904.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug in shutdown room admin API where it would fail if a user in the room hadn't consented to the privacy policy. diff --git a/changelog.d/4905.misc b/changelog.d/4905.misc deleted file mode 100644 index 0f00d5a3d5..0000000000 --- a/changelog.d/4905.misc +++ /dev/null @@ -1 +0,0 @@ -Log requests which are simulated by the unit tests. diff --git a/changelog.d/4908.bugfix b/changelog.d/4908.bugfix deleted file mode 100644 index d8c5babf0d..0000000000 --- a/changelog.d/4908.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where blocked world-readable rooms were still peekable. diff --git a/changelog.d/4912.misc b/changelog.d/4912.misc deleted file mode 100644 index f05a239187..0000000000 --- a/changelog.d/4912.misc +++ /dev/null @@ -1 +0,0 @@ -Allow newsfragments to end with exclamation marks. Exciting! diff --git a/changelog.d/4913.misc b/changelog.d/4913.misc deleted file mode 100644 index 9e835badc0..0000000000 --- a/changelog.d/4913.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor some more tests to use HomeserverTestCase. diff --git a/changelog.d/4917.misc b/changelog.d/4917.misc deleted file mode 100644 index 338d8a9a0c..0000000000 --- a/changelog.d/4917.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor out the state deltas portion of the user directory store and handler. diff --git a/changelog.d/4923.misc b/changelog.d/4923.misc deleted file mode 100644 index 8b5e1e3c81..0000000000 --- a/changelog.d/4923.misc +++ /dev/null @@ -1 +0,0 @@ -Fix nginx example in ACME doc. diff --git a/changelog.d/4927.feature b/changelog.d/4927.feature deleted file mode 100644 index 8d74262250..0000000000 --- a/changelog.d/4927.feature +++ /dev/null @@ -1 +0,0 @@ -Batch up outgoing read-receipts to reduce federation traffic. diff --git a/changelog.d/4928.misc b/changelog.d/4928.misc deleted file mode 100644 index 3a21752551..0000000000 --- a/changelog.d/4928.misc +++ /dev/null @@ -1 +0,0 @@ -Use an explicit dbname for postgres connections in the tests. diff --git a/changelog.d/4929.misc b/changelog.d/4929.misc deleted file mode 100644 index aaf02078b9..0000000000 --- a/changelog.d/4929.misc +++ /dev/null @@ -1 +0,0 @@ -Fix `ClientReplicationStreamProtocol.__str__()`. diff --git a/changelog.d/4931.feature b/changelog.d/4931.feature deleted file mode 100644 index 5d34d16800..0000000000 --- a/changelog.d/4931.feature +++ /dev/null @@ -1 +0,0 @@ -Add ability for password providers to login/register a user via 3PID (email, phone). \ No newline at end of file diff --git a/changelog.d/4944.feature b/changelog.d/4944.feature deleted file mode 100644 index 8f792b8890..0000000000 --- a/changelog.d/4944.feature +++ /dev/null @@ -1 +0,0 @@ -The user directory has been rewritten to make it faster, with less chance of falling behind on a large server. From f570916a3eb4088500e966182dea82647a5acac2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 27 Mar 2019 07:40:32 +0000 Subject: [PATCH 03/17] Add parse_row method to replication stream class This will allow individual stream classes to override how a row is parsed. --- synapse/replication/tcp/client.py | 5 +++-- synapse/replication/tcp/protocol.py | 2 +- synapse/replication/tcp/streams/_base.py | 15 +++++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 150975608f..206dc3b397 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -105,13 +105,14 @@ class ReplicationClientHandler(object): def on_rdata(self, stream_name, token, rows): """Called to handle a batch of replication data with a given stream token. - By default this just pokes the slave store. Can be overriden in subclasses to + By default this just pokes the slave store. Can be overridden in subclasses to handle more. Args: stream_name (str): name of the replication stream for this batch of rows token (int): stream token for this batch of rows - rows (list): a list of Stream.ROW_TYPE objects. + rows (list): a list of Stream.ROW_TYPE objects as returned by + Stream.parse_row. Returns: Deferred|None diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index 02e5bf6cc8..9daec2c995 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -605,7 +605,7 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol): inbound_rdata_count.labels(stream_name).inc() try: - row = STREAMS_MAP[stream_name].ROW_TYPE(*cmd.row) + row = STREAMS_MAP[stream_name].parse_row(cmd.row) except Exception: logger.exception( "[%s] Failed to parse RDATA: %r %r", diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index 18df89deed..25c3a23664 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -115,6 +115,21 @@ class Stream(object): ROW_TYPE = None # The type of the row _LIMITED = True # Whether the update function takes a limit + @classmethod + def parse_row(cls, row): + """Parse a row received over replication + + By default, assumes that the row data is an array object and passes its contents + to the constructor of the ROW_TYPE for this stream. + + Args: + row: row data from the incoming RDATA command, after json decoding + + Returns: + ROW_TYPE object for this stream + """ + return cls.ROW_TYPE(*row) + def __init__(self, hs): # The token from which we last asked for updates self.last_token = self.current_token() From 015b3622ebbea118baebc457227e355913a5702f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 27 Mar 2019 21:58:03 +0000 Subject: [PATCH 04/17] Skip building a ROW_TYPE when building updates We're about to turn it straight into a JSON object anyway so building a ROW_TYPE is a bit pointless, and reduces flexibility in the update_function. --- synapse/replication/tcp/streams/_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index 25c3a23664..13ab1bee05 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -112,7 +112,7 @@ class Stream(object): time it was called up until the point `advance_current_token` was called. """ NAME = None # The name of the stream - ROW_TYPE = None # The type of the row + ROW_TYPE = None # The type of the row. Used by the default impl of parse_row. _LIMITED = True # Whether the update function takes a limit @classmethod @@ -201,7 +201,7 @@ class Stream(object): from_token, current_token, ) - updates = [(row[0], self.ROW_TYPE(*row[1:])) for row in rows] + updates = [(row[0], row[1:]) for row in rows] # check we didn't get more rows than the limit. # doing it like this allows the update_function to be a generator. From a65763a5d6058dcb953970a566bbe91fc8d43daf Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 27 Mar 2019 22:04:01 +0000 Subject: [PATCH 05/17] changelog --- changelog.d/4954.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4954.misc diff --git a/changelog.d/4954.misc b/changelog.d/4954.misc new file mode 100644 index 0000000000..91f145950d --- /dev/null +++ b/changelog.d/4954.misc @@ -0,0 +1 @@ +Refactor replication row generation/parsing. From 1f6d6f918a57183083c2dd1bba6179373102b918 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 27 Mar 2019 15:18:28 +0000 Subject: [PATCH 06/17] Make EventStream rows have a type ... as a precursor to combining it with the CurrentStateDelta stream. --- synapse/app/synchrotron.py | 5 +- synapse/replication/slave/storage/events.py | 8 +- synapse/replication/tcp/protocol.py | 4 +- synapse/replication/tcp/streams/events.py | 98 ++++++++++++++++++--- 4 files changed, 98 insertions(+), 17 deletions(-) diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index 9163b56d86..5388def28a 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -48,6 +48,7 @@ from synapse.replication.slave.storage.receipts import SlavedReceiptsStore from synapse.replication.slave.storage.registration import SlavedRegistrationStore from synapse.replication.slave.storage.room import RoomStore from synapse.replication.tcp.client import ReplicationClientHandler +from synapse.replication.tcp.streams.events import EventsStreamEventRow from synapse.rest.client.v1 import events from synapse.rest.client.v1.initial_sync import InitialSyncRestServlet from synapse.rest.client.v1.room import RoomInitialSyncRestServlet @@ -369,7 +370,9 @@ class SyncReplicationHandler(ReplicationClientHandler): # We shouldn't get multiple rows per token for events stream, so # we don't need to optimise this for multiple rows. for row in rows: - event = yield self.store.get_event(row.event_id) + if row.type != EventsStreamEventRow.TypeId: + continue + event = yield self.store.get_event(row.data.event_id) extra_users = () if event.type == EventTypes.Member: extra_users = (event.state_key,) diff --git a/synapse/replication/slave/storage/events.py b/synapse/replication/slave/storage/events.py index 4830c68f35..c57385d92f 100644 --- a/synapse/replication/slave/storage/events.py +++ b/synapse/replication/slave/storage/events.py @@ -16,6 +16,7 @@ import logging from synapse.api.constants import EventTypes +from synapse.replication.tcp.streams.events import EventsStreamEventRow from synapse.storage.event_federation import EventFederationWorkerStore from synapse.storage.event_push_actions import EventPushActionsWorkerStore from synapse.storage.events_worker import EventsWorkerStore @@ -79,9 +80,12 @@ class SlavedEventStore(EventFederationWorkerStore, if stream_name == "events": self._stream_id_gen.advance(token) for row in rows: + if row.type != EventsStreamEventRow.TypeId: + continue + data = row.data self.invalidate_caches_for_event( - token, row.event_id, row.room_id, row.type, row.state_key, - row.redacts, + token, data.event_id, data.room_id, data.type, data.state_key, + data.redacts, backfilled=False, ) elif stream_name == "backfill": diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index 9daec2c995..b51590cf8f 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -42,8 +42,8 @@ indicate which side is sending, these are *not* included on the wire:: > POSITION backfill 1 > POSITION caches 1 > RDATA caches 2 ["get_user_by_id",["@01register-user:localhost:8823"],1490197670513] - > RDATA events 14 ["$149019767112vOHxz:localhost:8823", - "!AFDCvgApUmpdfVjIXm:localhost:8823","m.room.guest_access","",null] + > RDATA events 14 ["ev", ["$149019767112vOHxz:localhost:8823", + "!AFDCvgApUmpdfVjIXm:localhost:8823","m.room.guest_access","",null]] < PING 1490197675618 > ERROR server stopping * connection closed by server * diff --git a/synapse/replication/tcp/streams/events.py b/synapse/replication/tcp/streams/events.py index 511dd6bcc7..928028e893 100644 --- a/synapse/replication/tcp/streams/events.py +++ b/synapse/replication/tcp/streams/events.py @@ -13,28 +13,102 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from collections import namedtuple + +import attr + +from twisted.internet import defer from ._base import Stream -EventStreamRow = namedtuple("EventStreamRow", ( - "event_id", # str - "room_id", # str - "type", # str - "state_key", # str, optional - "redacts", # str, optional -)) + +"""Handling of the 'events' replication stream + +This stream contains rows of various types. Each row therefore contains a 'type' +identifier before the real data. For example:: + + RDATA events 12345 ["ev", ["$event:id", "!room:id", "m.type", null, null]] + +An "ev" row is sent for each new event. The fields in the data part are: + + * The new event id + * The room id for the event + * The type of the new event + * The state key of the event, for state events + * The event id of an event which is redacted by this event. + +""" + + +@attr.s(slots=True, frozen=True) +class EventsStreamRow(object): + """A parsed row from the events replication stream""" + type = attr.ib() # str: the TypeId of one of the *EventsStreamRows + data = attr.ib() # BaseEventsStreamRow + + +class BaseEventsStreamRow(object): + """Base class for rows to be sent in the events stream. + + Specifies how to identify, serialize and deserialize the different types. + """ + + TypeId = None # Unique string that ids the type. Must be overriden in sub classes. + + @classmethod + def from_data(cls, data): + """Parse the data from the replication stream into a row. + + By default we just call the constructor with the data list as arguments + + Args: + data: The value of the data object from the replication stream + """ + return cls(*data) + + +@attr.s(slots=True, frozen=True) +class EventsStreamEventRow(BaseEventsStreamRow): + TypeId = "ev" + + event_id = attr.ib() # str + room_id = attr.ib() # str + type = attr.ib() # str + state_key = attr.ib() # str, optional + redacts = attr.ib() # str, optional + + +TypeToRow = { + Row.TypeId: Row + for Row in ( + EventsStreamEventRow, + ) +} class EventsStream(Stream): """We received a new event, or an event went from being an outlier to not """ NAME = "events" - ROW_TYPE = EventStreamRow def __init__(self, hs): - store = hs.get_datastore() - self.current_token = store.get_current_events_token - self.update_function = store.get_all_new_forward_event_rows + self._store = hs.get_datastore() + self.current_token = self._store.get_current_events_token super(EventsStream, self).__init__(hs) + + @defer.inlineCallbacks + def update_function(self, from_token, current_token, limit=None): + event_rows = yield self._store.get_all_new_forward_event_rows( + from_token, current_token, limit, + ) + event_updates = ( + (row[0], EventsStreamEventRow.TypeId, row[1:]) + for row in event_rows + ) + defer.returnValue(event_updates) + + @classmethod + def parse_row(cls, row): + (typ, data) = row + data = TypeToRow[typ].from_data(data) + return EventsStreamRow(typ, data) From 4b91c313a94a4a89998e097e79a96a4423cf1b9f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 27 Mar 2019 16:15:59 +0000 Subject: [PATCH 07/17] Combine the CurrentStateDeltaStream into the EventStream --- synapse/app/user_dir.py | 17 ++++++----- synapse/replication/tcp/streams/__init__.py | 1 - synapse/replication/tcp/streams/_base.py | 21 ------------- synapse/replication/tcp/streams/events.py | 34 ++++++++++++++++++++- synapse/storage/events.py | 3 -- 5 files changed, 43 insertions(+), 33 deletions(-) diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py index d1ab9512cd..355f5aa71d 100644 --- a/synapse/app/user_dir.py +++ b/synapse/app/user_dir.py @@ -36,6 +36,10 @@ from synapse.replication.slave.storage.client_ips import SlavedClientIpStore from synapse.replication.slave.storage.events import SlavedEventStore from synapse.replication.slave.storage.registration import SlavedRegistrationStore from synapse.replication.tcp.client import ReplicationClientHandler +from synapse.replication.tcp.streams.events import ( + EventsStream, + EventsStreamCurrentStateRow, +) from synapse.rest.client.v2_alpha import user_directory from synapse.server import HomeServer from synapse.storage.engines import create_engine @@ -73,19 +77,18 @@ class UserDirectorySlaveStore( prefilled_cache=curr_state_delta_prefill, ) - self._current_state_delta_pos = events_max - def stream_positions(self): result = super(UserDirectorySlaveStore, self).stream_positions() - result["current_state_deltas"] = self._current_state_delta_pos return result def process_replication_rows(self, stream_name, token, rows): - if stream_name == "current_state_deltas": - self._current_state_delta_pos = token + if stream_name == EventsStream.NAME: + self._stream_id_gen.advance(token) for row in rows: + if row.type != EventsStreamCurrentStateRow.TypeId: + continue self._curr_state_delta_stream_cache.entity_has_changed( - row.room_id, token + row.data.room_id, token ) return super(UserDirectorySlaveStore, self).process_replication_rows( stream_name, token, rows @@ -170,7 +173,7 @@ class UserDirectoryReplicationHandler(ReplicationClientHandler): yield super(UserDirectoryReplicationHandler, self).on_rdata( stream_name, token, rows ) - if stream_name == "current_state_deltas": + if stream_name == EventsStream.NAME: run_in_background(self._notify_directory) @defer.inlineCallbacks diff --git a/synapse/replication/tcp/streams/__init__.py b/synapse/replication/tcp/streams/__init__.py index 5c715e3bfa..634f636dc9 100644 --- a/synapse/replication/tcp/streams/__init__.py +++ b/synapse/replication/tcp/streams/__init__.py @@ -44,7 +44,6 @@ STREAMS_MAP = { federation.FederationStream, _base.TagAccountDataStream, _base.AccountDataStream, - _base.CurrentStateDeltaStream, _base.GroupServerStream, ) } diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index 13ab1bee05..8971a6a22e 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -91,12 +91,6 @@ AccountDataStreamRow = namedtuple("AccountDataStream", ( "data_type", # str "data", # dict )) -CurrentStateDeltaStreamRow = namedtuple("CurrentStateDeltaStream", ( - "room_id", # str - "type", # str - "state_key", # str - "event_id", # str, optional -)) GroupsStreamRow = namedtuple("GroupsStreamRow", ( "group_id", # str "user_id", # str @@ -428,21 +422,6 @@ class AccountDataStream(Stream): defer.returnValue(results) -class CurrentStateDeltaStream(Stream): - """Current state for a room was changed - """ - NAME = "current_state_deltas" - ROW_TYPE = CurrentStateDeltaStreamRow - - def __init__(self, hs): - store = hs.get_datastore() - - self.current_token = store.get_max_current_state_delta_stream_id - self.update_function = store.get_all_updated_current_state_deltas - - super(CurrentStateDeltaStream, self).__init__(hs) - - class GroupServerStream(Stream): NAME = "groups" ROW_TYPE = GroupsStreamRow diff --git a/synapse/replication/tcp/streams/events.py b/synapse/replication/tcp/streams/events.py index 928028e893..e0f6e29248 100644 --- a/synapse/replication/tcp/streams/events.py +++ b/synapse/replication/tcp/streams/events.py @@ -13,6 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import heapq import attr @@ -26,6 +27,7 @@ from ._base import Stream This stream contains rows of various types. Each row therefore contains a 'type' identifier before the real data. For example:: + RDATA events batch ["state", ["!room:id", "m.type", "", "$event:id"]] RDATA events 12345 ["ev", ["$event:id", "!room:id", "m.type", null, null]] An "ev" row is sent for each new event. The fields in the data part are: @@ -36,6 +38,14 @@ An "ev" row is sent for each new event. The fields in the data part are: * The state key of the event, for state events * The event id of an event which is redacted by this event. +A "state" row is sent whenever the "current state" in a room changes. The fields in the +data part are: + + * The room id for the state change + * The event type of the state which has changed + * The state_key of the state which has changed + * The event id of the new state + """ @@ -77,10 +87,21 @@ class EventsStreamEventRow(BaseEventsStreamRow): redacts = attr.ib() # str, optional +@attr.s(slots=True, frozen=True) +class EventsStreamCurrentStateRow(BaseEventsStreamRow): + TypeId = "state" + + room_id = attr.ib() # str + type = attr.ib() # str + state_key = attr.ib() # str + event_id = attr.ib() # str, optional + + TypeToRow = { Row.TypeId: Row for Row in ( EventsStreamEventRow, + EventsStreamCurrentStateRow, ) } @@ -105,7 +126,18 @@ class EventsStream(Stream): (row[0], EventsStreamEventRow.TypeId, row[1:]) for row in event_rows ) - defer.returnValue(event_updates) + + state_rows = yield self._store.get_all_updated_current_state_deltas( + from_token, current_token, limit + ) + state_updates = ( + (row[0], EventsStreamCurrentStateRow.TypeId, row[1:]) + for row in state_rows + ) + + all_updates = heapq.merge(event_updates, state_updates) + + defer.returnValue(all_updates) @classmethod def parse_row(cls, row): diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 428300ea0a..0b0a4dcdd3 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -2287,9 +2287,6 @@ class EventsStore(StateGroupWorkerStore, EventFederationStore, EventsWorkerStore defer.returnValue((int(res["topological_ordering"]), int(res["stream_ordering"]))) - def get_max_current_state_delta_stream_id(self): - return self._stream_id_gen.get_current_token() - def get_all_updated_current_state_deltas(self, from_token, to_token, limit): def get_all_updated_current_state_deltas_txn(txn): sql = """ From 17d7bacbcffabec91257fae272840a203ddae98f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 27 Mar 2019 22:08:39 +0000 Subject: [PATCH 08/17] changelog --- changelog.d/4955.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4955.bugfix diff --git a/changelog.d/4955.bugfix b/changelog.d/4955.bugfix new file mode 100644 index 0000000000..e50e67383d --- /dev/null +++ b/changelog.d/4955.bugfix @@ -0,0 +1 @@ +Fix sync bug which made accepting invites unreliable in worker-mode synapses. From c7296bcb98e41caef25d59c85dfd043ea95a1835 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 28 Mar 2019 17:38:01 +0000 Subject: [PATCH 09/17] remove log line for password (#4965) Remove log line for password. --- changelog.d/4965.misc | 1 + synapse/rest/client/v1/admin.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 changelog.d/4965.misc diff --git a/changelog.d/4965.misc b/changelog.d/4965.misc new file mode 100644 index 0000000000..284c58b75e --- /dev/null +++ b/changelog.d/4965.misc @@ -0,0 +1 @@ +Remove log line for password via the admin API. diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index e788769639..1a26f5a1a6 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -647,8 +647,6 @@ class ResetPasswordRestServlet(ClientV1RestServlet): assert_params_in_dict(params, ["new_password"]) new_password = params['new_password'] - logger.info("new_password: %r", new_password) - yield self._set_password_handler.set_password( target_user_id, new_password, requester ) From aee4ea8ba881af566df6198b5d31f065a62a4459 Mon Sep 17 00:00:00 2001 From: Jurrie Overgoor <1213142+Jurrie@users.noreply.github.com> Date: Fri, 29 Mar 2019 10:15:13 +0100 Subject: [PATCH 10/17] Fix typo in TLS filenames Signed-off-by: Jurrie Overgoor <1213142+Jurrie@users.noreply.github.com> --- docker/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index 4b98b7fd75..a2cef668a6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -57,8 +57,8 @@ configuration file there. Multiple application services are supported. Synapse requires a valid TLS certificate. You can do one of the following: * Provide your own certificate and key (as - `${DATA_PATH}/${SYNAPSE_SERVER_NAME}.crt` and - `${DATA_PATH}/${SYNAPSE_SERVER_NAME}.key`, or elsewhere by providing an + `${DATA_PATH}/${SYNAPSE_SERVER_NAME}.tls.crt` and + `${DATA_PATH}/${SYNAPSE_SERVER_NAME}.tls.key`, or elsewhere by providing an entire config as `${SYNAPSE_CONFIG_PATH}`). * Use a reverse proxy to terminate incoming TLS, and forward the plain http From e0f219789d03e14a335e25832e4d39cbb705c955 Mon Sep 17 00:00:00 2001 From: Jurrie Overgoor <1213142+Jurrie@users.noreply.github.com> Date: Fri, 29 Mar 2019 10:15:39 +0100 Subject: [PATCH 11/17] Add -p argument for docker run command example Signed-off-by: Jurrie Overgoor <1213142+Jurrie@users.noreply.github.com> --- docker/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/README.md b/docker/README.md index a2cef668a6..44ade63f27 100644 --- a/docker/README.md +++ b/docker/README.md @@ -31,6 +31,7 @@ docker run \ --mount type=volume,src=synapse-data,dst=/data \ -e SYNAPSE_SERVER_NAME=my.matrix.host \ -e SYNAPSE_REPORT_STATS=yes \ + -p 8448:8448 \ matrixdotorg/synapse:latest ``` From 50b5f08740a8e351a4432a4f97c62a1ead299180 Mon Sep 17 00:00:00 2001 From: Jurrie Overgoor <1213142+Jurrie@users.noreply.github.com> Date: Fri, 29 Mar 2019 10:30:24 +0100 Subject: [PATCH 12/17] Add changelog.d entry --- changelog.d/4968.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4968.misc diff --git a/changelog.d/4968.misc b/changelog.d/4968.misc new file mode 100644 index 0000000000..7a7b69771c --- /dev/null +++ b/changelog.d/4968.misc @@ -0,0 +1 @@ +Fix typo in TLS filenames in docker/README.md. Also add the '-p' commandline option to the 'docker run' example. Contributed by Jurrie Overgoor. From 54a87a7b086048e2131a6d1ed00e25836ffa2995 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 1 Apr 2019 10:24:38 +0100 Subject: [PATCH 13/17] Collect room-version variations into one place (#4969) Collect all the things that make room-versions different to one another into one place, so that it's easier to define new room versions. --- changelog.d/4969.misc | 2 + synapse/api/constants.py | 40 -------- synapse/api/room_versions.py | 91 +++++++++++++++++++ synapse/event_auth.py | 20 ++-- synapse/events/__init__.py | 15 +-- synapse/events/builder.py | 36 +++++--- synapse/events/validator.py | 3 +- synapse/federation/federation_base.py | 16 ++-- synapse/federation/federation_client.py | 29 +++--- synapse/federation/federation_server.py | 3 +- synapse/federation/transport/server.py | 4 +- synapse/handlers/federation.py | 13 +-- synapse/handlers/message.py | 7 +- synapse/handlers/room.py | 11 +-- synapse/rest/client/v2_alpha/capabilities.py | 10 +- .../v2_alpha/room_upgrade_rest_servlet.py | 2 +- synapse/state/__init__.py | 16 +--- synapse/state/v1.py | 11 ++- synapse/storage/events_worker.py | 3 +- .../rest/client/v2_alpha/test_capabilities.py | 4 +- tests/state/test_v2.py | 7 +- tests/storage/test_redaction.py | 9 +- tests/storage/test_roommember.py | 5 +- tests/storage/test_state.py | 5 +- tests/test_event_auth.py | 10 +- tests/test_state.py | 5 +- tests/test_visibility.py | 8 +- tests/utils.py | 5 +- 28 files changed, 222 insertions(+), 168 deletions(-) create mode 100644 changelog.d/4969.misc create mode 100644 synapse/api/room_versions.py diff --git a/changelog.d/4969.misc b/changelog.d/4969.misc new file mode 100644 index 0000000000..e3a3214e6b --- /dev/null +++ b/changelog.d/4969.misc @@ -0,0 +1,2 @@ +Refactor room version definitions. + diff --git a/synapse/api/constants.py b/synapse/api/constants.py index f47c33a074..dc913feeee 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -102,46 +102,6 @@ class ThirdPartyEntityKind(object): LOCATION = "location" -class RoomVersions(object): - V1 = "1" - V2 = "2" - V3 = "3" - STATE_V2_TEST = "state-v2-test" - - -class RoomDisposition(object): - STABLE = "stable" - UNSTABLE = "unstable" - - -# the version we will give rooms which are created on this server -DEFAULT_ROOM_VERSION = RoomVersions.V1 - -# vdh-test-version is a placeholder to get room versioning support working and tested -# until we have a working v2. -KNOWN_ROOM_VERSIONS = { - RoomVersions.V1, - RoomVersions.V2, - RoomVersions.V3, - RoomVersions.STATE_V2_TEST, - RoomVersions.V3, -} - - -class EventFormatVersions(object): - """This is an internal enum for tracking the version of the event format, - independently from the room version. - """ - V1 = 1 - V2 = 2 - - -KNOWN_EVENT_FORMAT_VERSIONS = { - EventFormatVersions.V1, - EventFormatVersions.V2, -} - - ServerNoticeMsgType = "m.server_notice" ServerNoticeLimitReached = "m.server_notice.usage_limit_reached" diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py new file mode 100644 index 0000000000..e77abe1040 --- /dev/null +++ b/synapse/api/room_versions.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import attr + + +class EventFormatVersions(object): + """This is an internal enum for tracking the version of the event format, + independently from the room version. + """ + V1 = 1 # $id:server format + V2 = 2 # MSC1659-style $hash format: introduced for room v3 + + +KNOWN_EVENT_FORMAT_VERSIONS = { + EventFormatVersions.V1, + EventFormatVersions.V2, +} + + +class StateResolutionVersions(object): + """Enum to identify the state resolution algorithms""" + V1 = 1 # room v1 state res + V2 = 2 # MSC1442 state res: room v2 and later + + +class RoomDisposition(object): + STABLE = "stable" + UNSTABLE = "unstable" + + +@attr.s(slots=True, frozen=True) +class RoomVersion(object): + """An object which describes the unique attributes of a room version.""" + + identifier = attr.ib() # str; the identifier for this version + disposition = attr.ib() # str; one of the RoomDispositions + event_format = attr.ib() # int; one of the EventFormatVersions + state_res = attr.ib() # int; one of the StateResolutionVersions + + +class RoomVersions(object): + V1 = RoomVersion( + "1", + RoomDisposition.STABLE, + EventFormatVersions.V1, + StateResolutionVersions.V1, + ) + STATE_V2_TEST = RoomVersion( + "state-v2-test", + RoomDisposition.UNSTABLE, + EventFormatVersions.V1, + StateResolutionVersions.V2, + ) + V2 = RoomVersion( + "2", + RoomDisposition.STABLE, + EventFormatVersions.V1, + StateResolutionVersions.V2, + ) + V3 = RoomVersion( + "3", + RoomDisposition.STABLE, + EventFormatVersions.V2, + StateResolutionVersions.V2, + ) + + +# the version we will give rooms which are created on this server +DEFAULT_ROOM_VERSION = RoomVersions.V1 + + +KNOWN_ROOM_VERSIONS = { + v.identifier: v for v in ( + RoomVersions.V1, + RoomVersions.V2, + RoomVersions.V3, + RoomVersions.STATE_V2_TEST, + ) +} # type: dict[str, RoomVersion] diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 8f9e330da5..203490fc36 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -20,15 +20,9 @@ from signedjson.key import decode_verify_key_bytes from signedjson.sign import SignatureVerifyException, verify_signed_json from unpaddedbase64 import decode_base64 -from synapse.api.constants import ( - KNOWN_ROOM_VERSIONS, - EventFormatVersions, - EventTypes, - JoinRules, - Membership, - RoomVersions, -) +from synapse.api.constants import EventTypes, JoinRules, Membership from synapse.api.errors import AuthError, EventSizeError, SynapseError +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, EventFormatVersions from synapse.types import UserID, get_domain_from_id logger = logging.getLogger(__name__) @@ -452,16 +446,18 @@ def check_redaction(room_version, event, auth_events): if user_level >= redact_level: return False - if room_version in (RoomVersions.V1, RoomVersions.V2,): + v = KNOWN_ROOM_VERSIONS.get(room_version) + if not v: + raise RuntimeError("Unrecognized room version %r" % (room_version,)) + + if v.event_format == EventFormatVersions.V1: redacter_domain = get_domain_from_id(event.event_id) redactee_domain = get_domain_from_id(event.redacts) if redacter_domain == redactee_domain: return True - elif room_version == RoomVersions.V3: + else: event.internal_metadata.recheck_redaction = True return True - else: - raise RuntimeError("Unrecognized room version %r" % (room_version,)) raise AuthError( 403, diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index fafa135182..12056d5be2 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -21,7 +21,7 @@ import six from unpaddedbase64 import encode_base64 -from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventFormatVersions, RoomVersions +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, EventFormatVersions from synapse.util.caches import intern_dict from synapse.util.frozenutils import freeze @@ -351,18 +351,13 @@ def room_version_to_event_format(room_version): Returns: int """ - if room_version not in KNOWN_ROOM_VERSIONS: + v = KNOWN_ROOM_VERSIONS.get(room_version) + + if not v: # We should have already checked version, so this should not happen raise RuntimeError("Unrecognized room version %s" % (room_version,)) - if room_version in ( - RoomVersions.V1, RoomVersions.V2, RoomVersions.STATE_V2_TEST, - ): - return EventFormatVersions.V1 - elif room_version in (RoomVersions.V3,): - return EventFormatVersions.V2 - else: - raise RuntimeError("Unrecognized room version %s" % (room_version,)) + return v.event_format def event_type_from_format_version(format_version): diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 06e01be918..fba27177c7 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -17,21 +17,17 @@ import attr from twisted.internet import defer -from synapse.api.constants import ( +from synapse.api.constants import MAX_DEPTH +from synapse.api.room_versions import ( KNOWN_EVENT_FORMAT_VERSIONS, KNOWN_ROOM_VERSIONS, - MAX_DEPTH, EventFormatVersions, ) from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.types import EventID from synapse.util.stringutils import random_string -from . import ( - _EventInternalMetadata, - event_type_from_format_version, - room_version_to_event_format, -) +from . import _EventInternalMetadata, event_type_from_format_version @attr.s(slots=True, cmp=False, frozen=True) @@ -170,21 +166,34 @@ class EventBuilderFactory(object): def new(self, room_version, key_values): """Generate an event builder appropriate for the given room version + Deprecated: use for_room_version with a RoomVersion object instead + Args: - room_version (str): Version of the room that we're creating an - event builder for + room_version (str): Version of the room that we're creating an event builder + for key_values (dict): Fields used as the basis of the new event Returns: EventBuilder """ - - # There's currently only the one event version defined - if room_version not in KNOWN_ROOM_VERSIONS: + v = KNOWN_ROOM_VERSIONS.get(room_version) + if not v: raise Exception( "No event format defined for version %r" % (room_version,) ) + return self.for_room_version(v, key_values) + def for_room_version(self, room_version, key_values): + """Generate an event builder appropriate for the given room version + + Args: + room_version (synapse.api.room_versions.RoomVersion): + Version of the room that we're creating an event builder for + key_values (dict): Fields used as the basis of the new event + + Returns: + EventBuilder + """ return EventBuilder( store=self.store, state=self.state, @@ -192,7 +201,7 @@ class EventBuilderFactory(object): clock=self.clock, hostname=self.hostname, signing_key=self.signing_key, - format_version=room_version_to_event_format(room_version), + format_version=room_version.event_format, type=key_values["type"], state_key=key_values.get("state_key"), room_id=key_values["room_id"], @@ -222,7 +231,6 @@ def create_local_event_from_event_dict(clock, hostname, signing_key, FrozenEvent """ - # There's currently only the one event version defined if format_version not in KNOWN_EVENT_FORMAT_VERSIONS: raise Exception( "No event format defined for version %r" % (format_version,) diff --git a/synapse/events/validator.py b/synapse/events/validator.py index a072674b02..514273c792 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -15,8 +15,9 @@ from six import string_types -from synapse.api.constants import EventFormatVersions, EventTypes, Membership +from synapse.api.constants import EventTypes, Membership from synapse.api.errors import SynapseError +from synapse.api.room_versions import EventFormatVersions from synapse.types import EventID, RoomID, UserID diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index a7a2ec4523..dfe6b4aa5c 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -20,8 +20,9 @@ import six from twisted.internet import defer from twisted.internet.defer import DeferredList -from synapse.api.constants import MAX_DEPTH, EventTypes, Membership, RoomVersions +from synapse.api.constants import MAX_DEPTH, EventTypes, Membership from synapse.api.errors import Codes, SynapseError +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, EventFormatVersions from synapse.crypto.event_signing import check_event_content_hash from synapse.events import event_type_from_format_version from synapse.events.utils import prune_event @@ -274,9 +275,12 @@ def _check_sigs_on_pdus(keyring, room_version, pdus): # now let's look for events where the sender's domain is different to the # event id's domain (normally only the case for joins/leaves), and add additional # checks. Only do this if the room version has a concept of event ID domain - if room_version in ( - RoomVersions.V1, RoomVersions.V2, RoomVersions.STATE_V2_TEST, - ): + # (ie, the room version uses old-style non-hash event IDs). + v = KNOWN_ROOM_VERSIONS.get(room_version) + if not v: + raise RuntimeError("Unrecognized room version %s" % (room_version,)) + + if v.event_format == EventFormatVersions.V1: pdus_to_check_event_id = [ p for p in pdus_to_check if p.sender_domain != get_domain_from_id(p.pdu.event_id) @@ -289,10 +293,6 @@ def _check_sigs_on_pdus(keyring, room_version, pdus): for p, d in zip(pdus_to_check_event_id, more_deferreds): p.deferreds.append(d) - elif room_version in (RoomVersions.V3,): - pass # No further checks needed, as event IDs are hashes here - else: - raise RuntimeError("Unrecognized room version %s" % (room_version,)) # replace lists of deferreds with single Deferreds return [_flatten_deferred_list(p.deferreds) for p in pdus_to_check] diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 58e04d81ab..f3fc897a0a 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -25,12 +25,7 @@ from prometheus_client import Counter from twisted.internet import defer -from synapse.api.constants import ( - KNOWN_ROOM_VERSIONS, - EventTypes, - Membership, - RoomVersions, -) +from synapse.api.constants import EventTypes, Membership from synapse.api.errors import ( CodeMessageException, Codes, @@ -38,6 +33,11 @@ from synapse.api.errors import ( HttpResponseException, SynapseError, ) +from synapse.api.room_versions import ( + KNOWN_ROOM_VERSIONS, + EventFormatVersions, + RoomVersions, +) from synapse.events import builder, room_version_to_event_format from synapse.federation.federation_base import FederationBase, event_from_pdu_json from synapse.util import logcontext, unwrapFirstError @@ -570,7 +570,7 @@ class FederationClient(FederationBase): Deferred[tuple[str, FrozenEvent, int]]: resolves to a tuple of `(origin, event, event_format)` where origin is the remote homeserver which generated the event, and event_format is one of - `synapse.api.constants.EventFormatVersions`. + `synapse.api.room_versions.EventFormatVersions`. Fails with a ``SynapseError`` if the chosen remote server returns a 300/400 code. @@ -592,7 +592,7 @@ class FederationClient(FederationBase): # Note: If not supplied, the room version may be either v1 or v2, # however either way the event format version will be v1. - room_version = ret.get("room_version", RoomVersions.V1) + room_version = ret.get("room_version", RoomVersions.V1.identifier) event_format = room_version_to_event_format(room_version) pdu_dict = ret.get("event", None) @@ -695,7 +695,9 @@ class FederationClient(FederationBase): room_version = None for e in state: if (e.type, e.state_key) == (EventTypes.Create, ""): - room_version = e.content.get("room_version", RoomVersions.V1) + room_version = e.content.get( + "room_version", RoomVersions.V1.identifier + ) break if room_version is None: @@ -802,11 +804,10 @@ class FederationClient(FederationBase): raise err # Otherwise, we assume that the remote server doesn't understand - # the v2 invite API. - - if room_version in (RoomVersions.V1, RoomVersions.V2): - pass # We'll fall through - else: + # the v2 invite API. That's ok provided the room uses old-style event + # IDs. + v = KNOWN_ROOM_VERSIONS.get(room_version) + if v.event_format != EventFormatVersions.V1: raise SynapseError( 400, "User's homeserver does not support this room version", diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 81f3b4b1ff..df60828dba 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -25,7 +25,7 @@ from twisted.internet import defer from twisted.internet.abstract import isIPAddress from twisted.python import failure -from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventTypes, Membership +from synapse.api.constants import EventTypes, Membership from synapse.api.errors import ( AuthError, Codes, @@ -34,6 +34,7 @@ from synapse.api.errors import ( NotFoundError, SynapseError, ) +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.crypto.event_signing import compute_event_signature from synapse.events import room_version_to_event_format from synapse.federation.federation_base import FederationBase, event_from_pdu_json diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index efb6bdca48..452599e1a1 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -21,8 +21,8 @@ import re from twisted.internet import defer import synapse -from synapse.api.constants import RoomVersions from synapse.api.errors import Codes, FederationDeniedError, SynapseError +from synapse.api.room_versions import RoomVersions from synapse.api.urls import FEDERATION_V1_PREFIX, FEDERATION_V2_PREFIX from synapse.http.endpoint import parse_and_validate_server_name from synapse.http.server import JsonResource @@ -513,7 +513,7 @@ class FederationV1InviteServlet(BaseFederationServlet): # state resolution algorithm, and we don't use that for processing # invites content = yield self.handler.on_invite_request( - origin, content, room_version=RoomVersions.V1, + origin, content, room_version=RoomVersions.V1.identifier, ) # V1 federation API is defined to return a content of `[200, {...}]` diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 9eaf2d3e18..0684778882 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -29,13 +29,7 @@ from unpaddedbase64 import decode_base64 from twisted.internet import defer -from synapse.api.constants import ( - KNOWN_ROOM_VERSIONS, - EventTypes, - Membership, - RejectedReason, - RoomVersions, -) +from synapse.api.constants import EventTypes, Membership, RejectedReason from synapse.api.errors import ( AuthError, CodeMessageException, @@ -44,6 +38,7 @@ from synapse.api.errors import ( StoreError, SynapseError, ) +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.crypto.event_signing import compute_event_signature from synapse.event_auth import auth_types_for_event from synapse.events.validator import EventValidator @@ -1733,7 +1728,9 @@ class FederationHandler(BaseHandler): # invalid, and it would fail auth checks anyway. raise SynapseError(400, "No create event in state") - room_version = create_event.content.get("room_version", RoomVersions.V1) + room_version = create_event.content.get( + "room_version", RoomVersions.V1.identifier, + ) missing_auth_events = set() for e in itertools.chain(auth_events, state, [event]): diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 9b41c7b205..8bc7a7678a 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -22,7 +22,7 @@ from canonicaljson import encode_canonical_json, json from twisted.internet import defer from twisted.internet.defer import succeed -from synapse.api.constants import EventTypes, Membership, RoomVersions +from synapse.api.constants import EventTypes, Membership from synapse.api.errors import ( AuthError, Codes, @@ -30,6 +30,7 @@ from synapse.api.errors import ( NotFoundError, SynapseError, ) +from synapse.api.room_versions import RoomVersions from synapse.api.urls import ConsentURIBuilder from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator @@ -603,7 +604,9 @@ class EventCreationHandler(object): """ if event.is_state() and (event.type, event.state_key) == (EventTypes.Create, ""): - room_version = event.content.get("room_version", RoomVersions.V1) + room_version = event.content.get( + "room_version", RoomVersions.V1.identifier + ) else: room_version = yield self.store.get_room_version(event.room_id) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 67b15697fd..c3dcfec247 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -25,14 +25,9 @@ from six import iteritems, string_types from twisted.internet import defer -from synapse.api.constants import ( - DEFAULT_ROOM_VERSION, - KNOWN_ROOM_VERSIONS, - EventTypes, - JoinRules, - RoomCreationPreset, -) +from synapse.api.constants import EventTypes, JoinRules, RoomCreationPreset from synapse.api.errors import AuthError, Codes, NotFoundError, StoreError, SynapseError +from synapse.api.room_versions import DEFAULT_ROOM_VERSION, KNOWN_ROOM_VERSIONS from synapse.storage.state import StateFilter from synapse.types import RoomAlias, RoomID, RoomStreamToken, StreamToken, UserID from synapse.util import stringutils @@ -479,7 +474,7 @@ class RoomCreationHandler(BaseHandler): if ratelimit: yield self.ratelimit(requester) - room_version = config.get("room_version", DEFAULT_ROOM_VERSION) + room_version = config.get("room_version", DEFAULT_ROOM_VERSION.identifier) if not isinstance(room_version, string_types): raise SynapseError( 400, diff --git a/synapse/rest/client/v2_alpha/capabilities.py b/synapse/rest/client/v2_alpha/capabilities.py index 373f95126e..a868d06098 100644 --- a/synapse/rest/client/v2_alpha/capabilities.py +++ b/synapse/rest/client/v2_alpha/capabilities.py @@ -16,7 +16,7 @@ import logging from twisted.internet import defer -from synapse.api.constants import DEFAULT_ROOM_VERSION, RoomDisposition, RoomVersions +from synapse.api.room_versions import DEFAULT_ROOM_VERSION, KNOWN_ROOM_VERSIONS from synapse.http.servlet import RestServlet from ._base import client_v2_patterns @@ -48,12 +48,10 @@ class CapabilitiesRestServlet(RestServlet): response = { "capabilities": { "m.room_versions": { - "default": DEFAULT_ROOM_VERSION, + "default": DEFAULT_ROOM_VERSION.identifier, "available": { - RoomVersions.V1: RoomDisposition.STABLE, - RoomVersions.V2: RoomDisposition.STABLE, - RoomVersions.STATE_V2_TEST: RoomDisposition.UNSTABLE, - RoomVersions.V3: RoomDisposition.STABLE, + v.identifier: v.disposition + for v in KNOWN_ROOM_VERSIONS.values() }, }, "m.change_password": {"enabled": change_password}, diff --git a/synapse/rest/client/v2_alpha/room_upgrade_rest_servlet.py b/synapse/rest/client/v2_alpha/room_upgrade_rest_servlet.py index e6356101fd..3db7ff8d1b 100644 --- a/synapse/rest/client/v2_alpha/room_upgrade_rest_servlet.py +++ b/synapse/rest/client/v2_alpha/room_upgrade_rest_servlet.py @@ -17,8 +17,8 @@ import logging from twisted.internet import defer -from synapse.api.constants import KNOWN_ROOM_VERSIONS from synapse.api.errors import Codes, SynapseError +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.http.servlet import ( RestServlet, assert_params_in_dict, diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 68058f613c..52347fee34 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -24,7 +24,8 @@ from frozendict import frozendict from twisted.internet import defer -from synapse.api.constants import EventTypes, RoomVersions +from synapse.api.constants import EventTypes +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, StateResolutionVersions from synapse.events.snapshot import EventContext from synapse.state import v1, v2 from synapse.util.async_helpers import Linearizer @@ -603,22 +604,15 @@ def resolve_events_with_store(room_version, state_sets, event_map, state_res_sto Deferred[dict[(str, str), str]]: a map from (type, state_key) to event_id. """ - if room_version == RoomVersions.V1: + v = KNOWN_ROOM_VERSIONS[room_version] + if v.state_res == StateResolutionVersions.V1: return v1.resolve_events_with_store( state_sets, event_map, state_res_store.get_events, ) - elif room_version in ( - RoomVersions.STATE_V2_TEST, RoomVersions.V2, RoomVersions.V3, - ): + else: return v2.resolve_events_with_store( room_version, state_sets, event_map, state_res_store, ) - else: - # This should only happen if we added a version but forgot to add it to - # the list above. - raise Exception( - "No state resolution algorithm defined for version %r" % (room_version,) - ) @attr.s diff --git a/synapse/state/v1.py b/synapse/state/v1.py index 6d3afcae7c..29b4e86cfd 100644 --- a/synapse/state/v1.py +++ b/synapse/state/v1.py @@ -21,8 +21,9 @@ from six import iteritems, iterkeys, itervalues from twisted.internet import defer from synapse import event_auth -from synapse.api.constants import EventTypes, RoomVersions +from synapse.api.constants import EventTypes from synapse.api.errors import AuthError +from synapse.api.room_versions import RoomVersions logger = logging.getLogger(__name__) @@ -275,7 +276,9 @@ def _resolve_auth_events(events, auth_events): try: # The signatures have already been checked at this point event_auth.check( - RoomVersions.V1, event, auth_events, + RoomVersions.V1.identifier, + event, + auth_events, do_sig_check=False, do_size_check=False, ) @@ -291,7 +294,9 @@ def _resolve_normal_events(events, auth_events): try: # The signatures have already been checked at this point event_auth.check( - RoomVersions.V1, event, auth_events, + RoomVersions.V1.identifier, + event, + auth_events, do_sig_check=False, do_size_check=False, ) diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py index 1716be529a..53c8dc3903 100644 --- a/synapse/storage/events_worker.py +++ b/synapse/storage/events_worker.py @@ -21,8 +21,9 @@ from canonicaljson import json from twisted.internet import defer -from synapse.api.constants import EventFormatVersions, EventTypes +from synapse.api.constants import EventTypes from synapse.api.errors import NotFoundError +from synapse.api.room_versions import EventFormatVersions from synapse.events import FrozenEvent, event_type_from_format_version # noqa: F401 # these are only included to make the type annotations work from synapse.events.snapshot import EventContext # noqa: F401 diff --git a/tests/rest/client/v2_alpha/test_capabilities.py b/tests/rest/client/v2_alpha/test_capabilities.py index d3d43970fb..bbfc77e829 100644 --- a/tests/rest/client/v2_alpha/test_capabilities.py +++ b/tests/rest/client/v2_alpha/test_capabilities.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.api.constants import DEFAULT_ROOM_VERSION, KNOWN_ROOM_VERSIONS +from synapse.api.room_versions import DEFAULT_ROOM_VERSION, KNOWN_ROOM_VERSIONS from synapse.rest.client.v1 import admin, login from synapse.rest.client.v2_alpha import capabilities @@ -52,7 +52,7 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): for room_version in capabilities['m.room_versions']['available'].keys(): self.assertTrue(room_version in KNOWN_ROOM_VERSIONS, "" + room_version) self.assertEqual( - DEFAULT_ROOM_VERSION, capabilities['m.room_versions']['default'] + DEFAULT_ROOM_VERSION.identifier, capabilities['m.room_versions']['default'] ) def test_get_change_password_capabilities(self): diff --git a/tests/state/test_v2.py b/tests/state/test_v2.py index 9a5c816927..f448b01326 100644 --- a/tests/state/test_v2.py +++ b/tests/state/test_v2.py @@ -19,7 +19,8 @@ from six.moves import zip import attr -from synapse.api.constants import EventTypes, JoinRules, Membership, RoomVersions +from synapse.api.constants import EventTypes, JoinRules, Membership +from synapse.api.room_versions import RoomVersions from synapse.event_auth import auth_types_for_event from synapse.events import FrozenEvent from synapse.state.v2 import lexicographical_topological_sort, resolve_events_with_store @@ -539,7 +540,7 @@ class StateTestCase(unittest.TestCase): state_before = dict(state_at_event[prev_events[0]]) else: state_d = resolve_events_with_store( - RoomVersions.V2, + RoomVersions.V2.identifier, [state_at_event[n] for n in prev_events], event_map=event_map, state_res_store=TestStateResolutionStore(event_map), @@ -686,7 +687,7 @@ class SimpleParamStateTestCase(unittest.TestCase): # Test that we correctly handle passing `None` as the event_map state_d = resolve_events_with_store( - RoomVersions.V2, + RoomVersions.V2.identifier, [self.state_at_bob, self.state_at_charlie], event_map=None, state_res_store=TestStateResolutionStore(self.event_map), diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index 3957561b1e..0fc5019e9f 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -18,7 +18,8 @@ from mock import Mock from twisted.internet import defer -from synapse.api.constants import EventTypes, Membership, RoomVersions +from synapse.api.constants import EventTypes, Membership +from synapse.api.room_versions import RoomVersions from synapse.types import RoomID, UserID from tests import unittest @@ -51,7 +52,7 @@ class RedactionTestCase(unittest.TestCase): ): content = {"membership": membership} content.update(extra_content) - builder = self.event_builder_factory.new( + builder = self.event_builder_factory.for_room_version( RoomVersions.V1, { "type": EventTypes.Member, @@ -74,7 +75,7 @@ class RedactionTestCase(unittest.TestCase): def inject_message(self, room, user, body): self.depth += 1 - builder = self.event_builder_factory.new( + builder = self.event_builder_factory.for_room_version( RoomVersions.V1, { "type": EventTypes.Message, @@ -95,7 +96,7 @@ class RedactionTestCase(unittest.TestCase): @defer.inlineCallbacks def inject_redaction(self, room, event_id, user, reason): - builder = self.event_builder_factory.new( + builder = self.event_builder_factory.for_room_version( RoomVersions.V1, { "type": EventTypes.Redaction, diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index 7fa2f4fd70..063387863e 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -18,7 +18,8 @@ from mock import Mock from twisted.internet import defer -from synapse.api.constants import EventTypes, Membership, RoomVersions +from synapse.api.constants import EventTypes, Membership +from synapse.api.room_versions import RoomVersions from synapse.types import RoomID, UserID from tests import unittest @@ -49,7 +50,7 @@ class RoomMemberStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def inject_room_member(self, room, user, membership, replaces_state=None): - builder = self.event_builder_factory.new( + builder = self.event_builder_factory.for_room_version( RoomVersions.V1, { "type": EventTypes.Member, diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py index 99cd3e09eb..78e260a7fa 100644 --- a/tests/storage/test_state.py +++ b/tests/storage/test_state.py @@ -17,7 +17,8 @@ import logging from twisted.internet import defer -from synapse.api.constants import EventTypes, Membership, RoomVersions +from synapse.api.constants import EventTypes, Membership +from synapse.api.room_versions import RoomVersions from synapse.storage.state import StateFilter from synapse.types import RoomID, UserID @@ -48,7 +49,7 @@ class StateStoreTestCase(tests.unittest.TestCase): @defer.inlineCallbacks def inject_state_event(self, room, sender, typ, state_key, content): - builder = self.event_builder_factory.new( + builder = self.event_builder_factory.for_room_version( RoomVersions.V1, { "type": typ, diff --git a/tests/test_event_auth.py b/tests/test_event_auth.py index 7ee318e4e8..4c8f87e958 100644 --- a/tests/test_event_auth.py +++ b/tests/test_event_auth.py @@ -16,8 +16,8 @@ import unittest from synapse import event_auth -from synapse.api.constants import RoomVersions from synapse.api.errors import AuthError +from synapse.api.room_versions import RoomVersions from synapse.events import FrozenEvent @@ -37,7 +37,7 @@ class EventAuthTestCase(unittest.TestCase): # creator should be able to send state event_auth.check( - RoomVersions.V1, _random_state_event(creator), auth_events, + RoomVersions.V1.identifier, _random_state_event(creator), auth_events, do_sig_check=False, ) @@ -45,7 +45,7 @@ class EventAuthTestCase(unittest.TestCase): self.assertRaises( AuthError, event_auth.check, - RoomVersions.V1, + RoomVersions.V1.identifier, _random_state_event(joiner), auth_events, do_sig_check=False, @@ -74,7 +74,7 @@ class EventAuthTestCase(unittest.TestCase): self.assertRaises( AuthError, event_auth.check, - RoomVersions.V1, + RoomVersions.V1.identifier, _random_state_event(pleb), auth_events, do_sig_check=False, @@ -82,7 +82,7 @@ class EventAuthTestCase(unittest.TestCase): # king should be able to send state event_auth.check( - RoomVersions.V1, _random_state_event(king), auth_events, + RoomVersions.V1.identifier, _random_state_event(king), auth_events, do_sig_check=False, ) diff --git a/tests/test_state.py b/tests/test_state.py index e20c33322a..03e4810c2e 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -18,7 +18,8 @@ from mock import Mock from twisted.internet import defer from synapse.api.auth import Auth -from synapse.api.constants import EventTypes, Membership, RoomVersions +from synapse.api.constants import EventTypes, Membership +from synapse.api.room_versions import RoomVersions from synapse.events import FrozenEvent from synapse.state import StateHandler, StateResolutionHandler @@ -118,7 +119,7 @@ class StateGroupStore(object): self._event_to_state_group[event_id] = state_group def get_room_version(self, room_id): - return RoomVersions.V1 + return RoomVersions.V1.identifier class DictObj(dict): diff --git a/tests/test_visibility.py b/tests/test_visibility.py index 455db9f276..3bdb500514 100644 --- a/tests/test_visibility.py +++ b/tests/test_visibility.py @@ -17,7 +17,7 @@ import logging from twisted.internet import defer from twisted.internet.defer import succeed -from synapse.api.constants import RoomVersions +from synapse.api.room_versions import RoomVersions from synapse.events import FrozenEvent from synapse.visibility import filter_events_for_server @@ -124,7 +124,7 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase): @defer.inlineCallbacks def inject_visibility(self, user_id, visibility): content = {"history_visibility": visibility} - builder = self.event_builder_factory.new( + builder = self.event_builder_factory.for_room_version( RoomVersions.V1, { "type": "m.room.history_visibility", @@ -145,7 +145,7 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase): def inject_room_member(self, user_id, membership="join", extra_content={}): content = {"membership": membership} content.update(extra_content) - builder = self.event_builder_factory.new( + builder = self.event_builder_factory.for_room_version( RoomVersions.V1, { "type": "m.room.member", @@ -167,7 +167,7 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase): def inject_message(self, user_id, content=None): if content is None: content = {"body": "testytest", "msgtype": "m.text"} - builder = self.event_builder_factory.new( + builder = self.event_builder_factory.for_room_version( RoomVersions.V1, { "type": "m.room.message", diff --git a/tests/utils.py b/tests/utils.py index 615b9f8cca..cb75514851 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -27,8 +27,9 @@ from six.moves.urllib import parse as urlparse from twisted.internet import defer, reactor -from synapse.api.constants import EventTypes, RoomVersions +from synapse.api.constants import EventTypes from synapse.api.errors import CodeMessageException, cs_error +from synapse.api.room_versions import RoomVersions from synapse.config.homeserver import HomeServerConfig from synapse.federation.transport import server as federation_server from synapse.http.server import HttpServer @@ -671,7 +672,7 @@ def create_room(hs, room_id, creator_id): event_builder_factory = hs.get_event_builder_factory() event_creation_handler = hs.get_event_creation_handler() - builder = event_builder_factory.new( + builder = event_builder_factory.for_room_version( RoomVersions.V1, { "type": EventTypes.Create, From ed1ce0333cf34b5f7a3f83b6e684e503b44c0590 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 1 Apr 2019 12:53:08 +0100 Subject: [PATCH 14/17] convert rst link to md --- INSTALL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index de6893530d..a5c3c6efaa 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -384,7 +384,7 @@ To configure Synapse to expose an HTTPS port, you will need to edit `cert.pem`). For those of you upgrading your TLS certificate in readiness for Synapse 1.0, -please take a look at `our guide `_. +please take a look at [our guide](docs/MSC1711_certificates_FAQ.md#configuring-certificates-for-compatibility-with-synapse-100). ## Registering a user From bb925b1bd7d6c4ddc299f76105576538c002f28b Mon Sep 17 00:00:00 2001 From: manuroe Date: Mon, 1 Apr 2019 14:16:24 +0200 Subject: [PATCH 15/17] start.sh: Fix the --no-rate-limit option for messages and make it bypass rate limit on registration and login too. --- changelog.d/4981.bugfix | 1 + demo/start.sh | 20 +++++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 changelog.d/4981.bugfix diff --git a/changelog.d/4981.bugfix b/changelog.d/4981.bugfix new file mode 100644 index 0000000000..e51b45eec0 --- /dev/null +++ b/changelog.d/4981.bugfix @@ -0,0 +1 @@ +start.sh: Fix the --no-rate-limit option for messages and make it bypass rate limit on registration and login too. \ No newline at end of file diff --git a/demo/start.sh b/demo/start.sh index dcc4d6f4fa..c4a1328a6f 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -27,17 +27,27 @@ for port in 8080 8081 8082; do --config-path "$DIR/etc/$port.config" \ --report-stats no + printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config + echo 'enable_registration: true' >> $DIR/etc/$port.config + # Check script parameters if [ $# -eq 1 ]; then if [ $1 = "--no-rate-limit" ]; then - # Set high limits in config file to disable rate limiting - perl -p -i -e 's/rc_messages_per_second.*/rc_messages_per_second: 1000/g' $DIR/etc/$port.config - perl -p -i -e 's/rc_message_burst_count.*/rc_message_burst_count: 1000/g' $DIR/etc/$port.config + # messages rate limit + echo 'rc_messages_per_second: 1000' >> $DIR/etc/$port.config + echo 'rc_message_burst_count: 1000' >> $DIR/etc/$port.config + + # registration rate limit + printf 'rc_registration:\n per_second: 1000\n burst_count: 1000\n' >> $DIR/etc/$port.config + + # login rate limit + echo 'rc_login:' >> $DIR/etc/$port.config + printf ' address:\n per_second: 1000\n burst_count: 1000\n' >> $DIR/etc/$port.config + printf ' account:\n per_second: 1000\n burst_count: 1000\n' >> $DIR/etc/$port.config + printf ' failed_attempts:\n per_second: 1000\n burst_count: 1000\n' >> $DIR/etc/$port.config fi fi - perl -p -i -e 's/^enable_registration:.*/enable_registration: true/g' $DIR/etc/$port.config - if ! grep -F "full_twisted_stacktraces" -q $DIR/etc/$port.config; then echo "full_twisted_stacktraces: true" >> $DIR/etc/$port.config fi From 35442efb758960d72927d8bd698be657eb0e3037 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 1 Apr 2019 12:49:03 +0000 Subject: [PATCH 16/17] 0.99.3 --- CHANGES.md | 6 ++++++ debian/changelog | 8 ++++++-- synapse/__init__.py | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b13a324037..490c2021e0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +Synapse 0.99.3 (2019-04-01) +=========================== + +No significant changes. + + Synapse 0.99.3rc1 (2019-03-27) ============================== diff --git a/debian/changelog b/debian/changelog index d84931ec03..03df2e1c00 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,12 @@ -matrix-synapse-py3 (0.99.3) UNRELEASED; urgency=medium +matrix-synapse-py3 (0.99.3) stable; urgency=medium + [ Richard van der Hoff ] * Fix warning during preconfiguration. (Fixes: #4819) - -- Richard van der Hoff Thu, 07 Mar 2019 07:17:00 +0000 + [ Synapse Packaging team ] + * New synapse release 0.99.3. + + -- Synapse Packaging team Mon, 01 Apr 2019 12:48:21 +0000 matrix-synapse-py3 (0.99.2) stable; urgency=medium diff --git a/synapse/__init__.py b/synapse/__init__.py index 4a3a3ba236..6bb5a8b24d 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.99.3rc1" +__version__ = "0.99.3" From 4c552ed78aa367c37b262af15317b943fc266776 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 1 Apr 2019 17:42:18 +0100 Subject: [PATCH 17/17] Neilj/fix threepid auth check (with tests) (#4474) test threepid checking --- changelog.d/4474.misc | 1 + tests/config/test_server.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 changelog.d/4474.misc create mode 100644 tests/config/test_server.py diff --git a/changelog.d/4474.misc b/changelog.d/4474.misc new file mode 100644 index 0000000000..4b882d60be --- /dev/null +++ b/changelog.d/4474.misc @@ -0,0 +1 @@ +Add test to verify threepid auth check added in #4435. diff --git a/tests/config/test_server.py b/tests/config/test_server.py new file mode 100644 index 0000000000..f5836d73ac --- /dev/null +++ b/tests/config/test_server.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.config.server import is_threepid_reserved + +from tests import unittest + + +class ServerConfigTestCase(unittest.TestCase): + + def test_is_threepid_reserved(self): + user1 = {'medium': 'email', 'address': 'user1@example.com'} + user2 = {'medium': 'email', 'address': 'user2@example.com'} + user3 = {'medium': 'email', 'address': 'user3@example.com'} + user1_msisdn = {'medium': 'msisdn', 'address': '447700000000'} + config = [user1, user2] + + self.assertTrue(is_threepid_reserved(config, user1)) + self.assertFalse(is_threepid_reserved(config, user3)) + self.assertFalse(is_threepid_reserved(config, user1_msisdn))