From 474c9aadbe542440bb7cd764a654518b5c36f851 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 15 Feb 2017 19:32:20 +0000 Subject: [PATCH 01/16] Allow forgetting rooms you're banned from --- synapse/handlers/room_member.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index b2806555cf..2052d6d05f 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -719,7 +719,9 @@ class RoomMemberHandler(BaseHandler): ) membership = member.membership if member else None - if membership is not None and membership != Membership.LEAVE: + if membership is not None and membership not in [ + Membership.LEAVE, Membership.BAN + ]: raise SynapseError(400, "User %s in room %s" % ( user_id, room_id )) From b4017539d49b16e2d91ca8e7d312c9da4dbf3ae7 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Feb 2017 10:42:57 +0000 Subject: [PATCH 02/16] Make the pushers lang field column longer To accommodate things like zh-Hans-CN Fixes https://github.com/vector-im/riot-ios/issues/1031 --- synapse/storage/schema/delta/40/pushers.sql | 39 +++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 synapse/storage/schema/delta/40/pushers.sql diff --git a/synapse/storage/schema/delta/40/pushers.sql b/synapse/storage/schema/delta/40/pushers.sql new file mode 100644 index 0000000000..7e34927efa --- /dev/null +++ b/synapse/storage/schema/delta/40/pushers.sql @@ -0,0 +1,39 @@ +/* Copyright 2017 Vector Creations Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS pushers2 ( + id BIGINT PRIMARY KEY, + user_name TEXT NOT NULL, + access_token BIGINT DEFAULT NULL, + profile_tag VARCHAR(32) NOT NULL, + kind VARCHAR(8) NOT NULL, + app_id VARCHAR(64) NOT NULL, + app_display_name VARCHAR(64) NOT NULL, + device_display_name VARCHAR(128) NOT NULL, + pushkey TEXT NOT NULL, + ts BIGINT NOT NULL, + lang VARCHAR(16), + data TEXT, + last_stream_ordering INTEGER, + last_success BIGINT, + failing_since BIGINT, + UNIQUE (app_id, pushkey, user_name) +); + +INSERT INTO pushers2 SELECT * FROM PUSHERS; + +DROP TABLE PUSHERS; + +ALTER TABLE pushers2 RENAME TO pushers; From 4aa29508af2ac9ac07f91a33d5d682b4b5815a9c Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Feb 2017 10:51:49 +0000 Subject: [PATCH 03/16] Use TEXT rather than VARCHAR While we're changing anyway --- synapse/storage/schema/delta/40/pushers.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/storage/schema/delta/40/pushers.sql b/synapse/storage/schema/delta/40/pushers.sql index 7e34927efa..054a223f14 100644 --- a/synapse/storage/schema/delta/40/pushers.sql +++ b/synapse/storage/schema/delta/40/pushers.sql @@ -17,14 +17,14 @@ CREATE TABLE IF NOT EXISTS pushers2 ( id BIGINT PRIMARY KEY, user_name TEXT NOT NULL, access_token BIGINT DEFAULT NULL, - profile_tag VARCHAR(32) NOT NULL, - kind VARCHAR(8) NOT NULL, - app_id VARCHAR(64) NOT NULL, - app_display_name VARCHAR(64) NOT NULL, - device_display_name VARCHAR(128) NOT NULL, + profile_tag TEXT NOT NULL, + kind TEXT NOT NULL, + app_id TEXT NOT NULL, + app_display_name TEXT NOT NULL, + device_display_name TEXT NOT NULL, pushkey TEXT NOT NULL, ts BIGINT NOT NULL, - lang VARCHAR(16), + lang TEXT, data TEXT, last_stream_ordering INTEGER, last_success BIGINT, From 5aae844e608554bdf5fd4487c729dd978b6eeffc Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 17 Feb 2017 12:48:53 +0000 Subject: [PATCH 04/16] Add an example log_config file --- contrib/example_log_config.yaml | 48 +++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 contrib/example_log_config.yaml diff --git a/contrib/example_log_config.yaml b/contrib/example_log_config.yaml new file mode 100644 index 0000000000..c582af42d1 --- /dev/null +++ b/contrib/example_log_config.yaml @@ -0,0 +1,48 @@ +# Example log_config file for synapse. To enable, point `log_config` to it in +# `homeserver.yaml`, and restart synapse. +# +# This configuration will produce similar results to the defaults within +# synapse, but can be edited to give more flexibility. + +version: 1 + +formatters: + fmt: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s' + +filters: + context: + (): synapse.util.logcontext.LoggingContextFilter + request: "" + +handlers: + # example output to console + console: + class: logging.StreamHandler + filters: [context] + + # example output to file - to enable, edit 'root' config below. + file: + class: logging.handlers.RotatingFileHandler + formatter: fmt + filename: /var/log/synapse/homeserver.log + maxBytes: 100000000 + backupCount: 3 + filters: [context] + + +root: + level: INFO + handlers: [console] # to use file handler instead, switch to [file] + +loggers: + synapse: + level: INFO + + synapse.storage: + level: INFO + + # example of enabling debugging for a component: + # + # synapse.federation.transport.server: + # level: DEBUG \ No newline at end of file From 66eb0bd548afd87f08f7e6e994774fe29589babe Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 17 Feb 2017 12:55:36 +0000 Subject: [PATCH 05/16] Update example_log_config.yaml add trailing NL --- contrib/example_log_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/example_log_config.yaml b/contrib/example_log_config.yaml index c582af42d1..7f7c8ba588 100644 --- a/contrib/example_log_config.yaml +++ b/contrib/example_log_config.yaml @@ -45,4 +45,4 @@ loggers: # example of enabling debugging for a component: # # synapse.federation.transport.server: - # level: DEBUG \ No newline at end of file + # level: DEBUG From 699be7d1be5238172677c98078c2a973ab28fb9d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 18 Feb 2017 14:41:31 +0000 Subject: [PATCH 06/16] Fix up notif rotation --- synapse/storage/event_push_actions.py | 36 ++++++++++++++++----------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 808c9d22fc..52d2bd3ffb 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -168,19 +168,13 @@ class EventPushActionsStore(SQLBaseStore): row = txn.fetchone() notify_count = row[0] if row else 0 - summary_notif_count = self._simple_select_one_onecol_txn( - txn, - table="event_push_summary", - keyvalues={ - "user_id": user_id, - "room_id": room_id, - }, - retcol="notif_count", - allow_none=True, - ) - - if summary_notif_count: - notify_count += summary_notif_count + txn.execute(""" + SELECT notif_count FROM event_push_summary + WHERE room_id = ? AND user_id = ? AND stream_ordering > ? + """, (room_id, user_id, stream_ordering,)) + rows = txn.fetchall() + if rows: + notify_count += rows[0][0] # Now get the number of highlights sql = ( @@ -645,12 +639,20 @@ class EventPushActionsStore(SQLBaseStore): # We want to make sure that we only ever do this one at a time self.database_engine.lock_table(txn, "event_push_summary") + old_rotate_stream_ordering = self._simple_select_one_onecol_txn( + txn, + table="event_push_summary_stream_ordering", + keyvalues={}, + retcol="stream_ordering", + ) + # We don't to try and rotate millions of rows at once, so we cap the # maximum stream ordering we'll rotate before. txn.execute(""" SELECT stream_ordering FROM event_push_actions + WHERE stream_ordering > ? ORDER BY stream_ordering ASC LIMIT 1 OFFSET 50000 - """) + """, (old_rotate_stream_ordering,)) stream_row = txn.fetchone() if stream_row: offset_stream_ordering, = stream_row @@ -662,6 +664,8 @@ class EventPushActionsStore(SQLBaseStore): rotate_to_stream_ordering = self.stream_ordering_day_ago caught_up = True + logger.info("Rotating notifications up to: %s", rotate_to_stream_ordering) + self._rotate_notifs_before_txn(txn, rotate_to_stream_ordering) # We have caught up iff we were limited by `stream_ordering_day_ago` @@ -695,6 +699,8 @@ class EventPushActionsStore(SQLBaseStore): txn.execute(sql, (old_rotate_stream_ordering, rotate_to_stream_ordering,)) rows = txn.fetchall() + logger.info("Rotating notifications, handling %d rows", len(rows)) + # If the `old.user_id` above is NULL then we know there isn't already an # entry in the table, so we simply insert it. Otherwise we update the # existing table. @@ -726,6 +732,8 @@ class EventPushActionsStore(SQLBaseStore): (old_rotate_stream_ordering, rotate_to_stream_ordering,) ) + logger.info("Rotating notifications, deleted %s push actions", txn.rowcount) + txn.execute( "UPDATE event_push_summary_stream_ordering SET stream_ordering = ?", (rotate_to_stream_ordering,) From 7efb38d1dd058dcb9e7051aeaf2019520101576d Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 19 Feb 2017 22:55:48 +0000 Subject: [PATCH 07/16] Update metrics-howto.rst --- docs/metrics-howto.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/metrics-howto.rst b/docs/metrics-howto.rst index ca10799b00..56e69c1859 100644 --- a/docs/metrics-howto.rst +++ b/docs/metrics-howto.rst @@ -9,11 +9,13 @@ How to monitor Synapse metrics using Prometheus prometheus itself defaults to 9090, so starting just above that for locally monitored services seems reasonable. E.g. 9092: - Add to homeserver.yaml + Add to homeserver.yaml:: metrics_port: 9092 - Restart synapse + Also ensure that ``enable_metrics`` is set to ``True``. + + Restart synapse. 3: Add a prometheus target for synapse. It needs to set the ``metrics_path`` to a non-default value:: From e556aefe0a0ac9e09429aed4436cd34aa6e586c0 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 19 Feb 2017 23:06:08 +0000 Subject: [PATCH 08/16] Update metrics-howto.rst --- docs/metrics-howto.rst | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/docs/metrics-howto.rst b/docs/metrics-howto.rst index 56e69c1859..231c680786 100644 --- a/docs/metrics-howto.rst +++ b/docs/metrics-howto.rst @@ -1,24 +1,27 @@ How to monitor Synapse metrics using Prometheus =============================================== -1: Install prometheus: - Follow instructions at http://prometheus.io/docs/introduction/install/ +1. Install prometheus: -2: Enable synapse metrics: - Simply setting a (local) port number will enable it. Pick a port. - prometheus itself defaults to 9090, so starting just above that for - locally monitored services seems reasonable. E.g. 9092: + Follow instructions at http://prometheus.io/docs/introduction/install/ - Add to homeserver.yaml:: +2. Enable synapse metrics: - metrics_port: 9092 + Simply setting a (local) port number will enable it. Pick a port. + prometheus itself defaults to 9090, so starting just above that for + locally monitored services seems reasonable. E.g. 9092: - Also ensure that ``enable_metrics`` is set to ``True``. + Add to homeserver.yaml:: + + metrics_port: 9092 + + Also ensure that ``enable_metrics`` is set to ``True``. - Restart synapse. + Restart synapse. -3: Add a prometheus target for synapse. It needs to set the ``metrics_path`` - to a non-default value:: +3. Add a prometheus target for synapse. + + It needs to set the ``metrics_path`` to a non-default value:: - job_name: "synapse" metrics_path: "/_synapse/metrics" @@ -26,6 +29,9 @@ How to monitor Synapse metrics using Prometheus - targets: "my.server.here:9092" + If your prometheus is older than 1.5.2, you will need to replace + ``static_configs`` in the above with ``target_groups``. + Standard Metric Names --------------------- From 6184f6fcbcc3c37e915596a6e2013f6983c9e260 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 19 Feb 2017 23:06:45 +0000 Subject: [PATCH 09/16] Update metrics-howto.rst --- docs/metrics-howto.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/metrics-howto.rst b/docs/metrics-howto.rst index 231c680786..7390ab85c9 100644 --- a/docs/metrics-howto.rst +++ b/docs/metrics-howto.rst @@ -31,6 +31,8 @@ How to monitor Synapse metrics using Prometheus If your prometheus is older than 1.5.2, you will need to replace ``static_configs`` in the above with ``target_groups``. + + Restart prometheus. Standard Metric Names --------------------- From 7f026792e12b7ca0d24ca4c8ed9ed239bf8a99ac Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 20 Feb 2017 14:54:50 +0000 Subject: [PATCH 10/16] Fix /context/ visibiltiy rules --- synapse/handlers/room.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 7e7671c9a2..73bc73a45e 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -375,12 +375,15 @@ class RoomContextHandler(BaseHandler): now_token = yield self.hs.get_event_sources().get_current_token() + users = yield self.store.get_users_in_room(room_id) + is_peeking = user.to_string() not in users + def filter_evts(events): return filter_events_for_client( self.store, user.to_string(), events, - is_peeking=is_guest + is_peeking=is_peeking ) event = yield self.store.get_event(event_id, get_prev_content=True, From 17673404fbb1ca7b84efe7ca12955754bf366b09 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 20 Feb 2017 15:02:01 +0000 Subject: [PATCH 11/16] Remove unused param --- synapse/handlers/room.py | 2 +- synapse/rest/client/v1/room.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 73bc73a45e..99cb7db0db 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -356,7 +356,7 @@ class RoomCreationHandler(BaseHandler): class RoomContextHandler(BaseHandler): @defer.inlineCallbacks - def get_event_context(self, user, room_id, event_id, limit, is_guest): + def get_event_context(self, user, room_id, event_id, limit): """Retrieves events, pagination tokens and state around a given event in a room. diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 728e3df0e3..90242a6bac 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -505,7 +505,6 @@ class RoomEventContext(ClientV1RestServlet): room_id, event_id, limit, - requester.is_guest, ) if not results: From efff39c03013f4cd5303466440d84484f338cb57 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 20 Feb 2017 14:54:50 +0000 Subject: [PATCH 12/16] Fix /context/ visibiltiy rules --- synapse/handlers/room.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 7e7671c9a2..73bc73a45e 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -375,12 +375,15 @@ class RoomContextHandler(BaseHandler): now_token = yield self.hs.get_event_sources().get_current_token() + users = yield self.store.get_users_in_room(room_id) + is_peeking = user.to_string() not in users + def filter_evts(events): return filter_events_for_client( self.store, user.to_string(), events, - is_peeking=is_guest + is_peeking=is_peeking ) event = yield self.store.get_event(event_id, get_prev_content=True, From 6226a27bf8ff4a427888110dd9d05ea969885003 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 20 Feb 2017 15:02:01 +0000 Subject: [PATCH 13/16] Remove unused param --- synapse/handlers/room.py | 2 +- synapse/rest/client/v1/room.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 73bc73a45e..99cb7db0db 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -356,7 +356,7 @@ class RoomCreationHandler(BaseHandler): class RoomContextHandler(BaseHandler): @defer.inlineCallbacks - def get_event_context(self, user, room_id, event_id, limit, is_guest): + def get_event_context(self, user, room_id, event_id, limit): """Retrieves events, pagination tokens and state around a given event in a room. diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 2ebf5e59a0..6554f57df1 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -505,7 +505,6 @@ class RoomEventContext(ClientV1RestServlet): room_id, event_id, limit, - requester.is_guest, ) if not results: From 0c4cf9372b3200325109dc2e4975197aa9b7b7ca Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 20 Feb 2017 16:43:29 +0000 Subject: [PATCH 14/16] Fix a race in transaction queue It was theoretically possible for a PDU to get queued and not sent for ages. On closer inspection I think there were bigger problems elsewhere, but we might as well fix this since it's easy. --- synapse/federation/transaction_queue.py | 30 +++++++++++++++++-------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index bb3d9258a6..90235ff098 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -303,18 +303,10 @@ class TransactionQueue(object): try: self.pending_transactions[destination] = 1 + # XXX: what's this for? yield run_on_reactor() while True: - pending_pdus = self.pending_pdus_by_dest.pop(destination, []) - pending_edus = self.pending_edus_by_dest.pop(destination, []) - pending_presence = self.pending_presence_by_dest.pop(destination, {}) - pending_failures = self.pending_failures_by_dest.pop(destination, []) - - pending_edus.extend( - self.pending_edus_keyed_by_dest.pop(destination, {}).values() - ) - limiter = yield get_retry_limiter( destination, self.clock, @@ -326,6 +318,24 @@ class TransactionQueue(object): yield self._get_new_device_messages(destination) ) + # BEGIN CRITICAL SECTION + # + # In order to avoid a race condition, we need to make sure that + # the following code (from popping the queues up to the point + # where we decide if we actually have any pending messages) is + # atomic - otherwise new PDUs or EDUs might arrive in the + # meantime, but not get sent because we hold the + # pending_transactions flag. + + pending_pdus = self.pending_pdus_by_dest.pop(destination, []) + pending_edus = self.pending_edus_by_dest.pop(destination, []) + pending_presence = self.pending_presence_by_dest.pop(destination, {}) + pending_failures = self.pending_failures_by_dest.pop(destination, []) + + pending_edus.extend( + self.pending_edus_keyed_by_dest.pop(destination, {}).values() + ) + pending_edus.extend(device_message_edus) if pending_presence: pending_edus.append( @@ -355,6 +365,8 @@ class TransactionQueue(object): ) return + # END CRITICAL SECTION + success = yield self._send_new_transaction( destination, pending_pdus, pending_edus, pending_failures, limiter=limiter, From 30ecfef5a394a6ce1537c171b230d51d0e79197a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 21 Feb 2017 13:35:30 +0000 Subject: [PATCH 15/16] Bump version and changelog --- CHANGES.rst | 7 +++++++ synapse/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 22aa7cb9b4..c8856d9575 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,10 @@ +Changes in synapse v0.19.2 (2017-02-20) +======================================= + +* Fix bug with event visibility check in /context/ API. Thanks to Tokodomo for + pointing it out! (PR #1929) + + Changes in synapse v0.19.1 (2017-02-09) ======================================= diff --git a/synapse/__init__.py b/synapse/__init__.py index da8ef90a77..ff251ce597 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.19.1" +__version__ = "0.19.2" From b7442c3e2b333616093af76ab7848fa82b6490a9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 21 Feb 2017 13:59:25 +0000 Subject: [PATCH 16/16] Store looping call --- synapse/storage/event_push_actions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 52d2bd3ffb..cde141e20d 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -84,7 +84,9 @@ class EventPushActionsStore(SQLBaseStore): ) self._doing_notif_rotation = False - self._clock.looping_call(self._rotate_notifs, 30 * 60 * 1000) + self._rotate_notif_loop = self._clock.looping_call( + self._rotate_notifs, 30 * 60 * 1000 + ) def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples): """