From 3dac27a8a9846a892284971f71e05c2440225484 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Oct 2014 14:54:26 +0100 Subject: [PATCH 001/227] Storage for pdu signatures --- synapse/storage/schema/signatures.sql | 36 +++++++++++ synapse/storage/signatures.py | 90 +++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 synapse/storage/schema/signatures.sql create mode 100644 synapse/storage/signatures.py diff --git a/synapse/storage/schema/signatures.sql b/synapse/storage/schema/signatures.sql new file mode 100644 index 0000000000..ba3bbb5471 --- /dev/null +++ b/synapse/storage/schema/signatures.sql @@ -0,0 +1,36 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS pdu_hashes ( + pdu_id TEXT, + origin TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm) +); + +CREATE INDEX IF NOT EXISTS pdu_hashes_id ON pdu_hashes (pdu_id, origin); + +CREATE TABLE IF NOT EXISTS pdu_origin_signatures ( + pdu_id TEXT, + origin TEXT, + key_id TEXT, + signature BLOB, + CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm) +); + +CREATE INDEX IF NOT EXISTS pdu_origin_signatures_id ON pdu_origin_signatures ( + pdu_id, origin, +); diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py new file mode 100644 index 0000000000..bb860f09f0 --- /dev/null +++ b/synapse/storage/signatures.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from _base import SQLBaseStore + +from twisted.internet import defer + + +class SignatureStore(SQLBaseStore): + """Persistence for PDU signatures and hashes""" + + def _get_pdu_hashes_txn(self, txn, pdu_id, origin): + """Get all the hashes for a given PDU. + Args: + txn (cursor): + pdu_id (str): Id for the PDU. + origin (str): origin of the PDU. + Returns: + A dict of algorithm -> hash. + """ + query = ( + "SELECT algorithm, hash" + " FROM pdu_hashes" + " WHERE pdu_id = ? and origin = ?" + ) + txn.execute(query, (pdu_id, origin)) + return dict(txn.fetchall()) + + def _store_pdu_hash_txn(self, txn, pdu_id, origin, algorithm, hash_bytes): + """Store a hash for a PDU + Args: + txn (cursor): + pdu_id (str): Id for the PDU. + origin (str): origin of the PDU. + algorithm (str): Hashing algorithm. + hash_bytes (bytes): Hash function output bytes. + """ + self._simple_insert_txn(self, txn, "pdu_hashes", { + "pdu_id": pdu_id, + "origin": origin, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }) + + def _get_pdu_origin_signatures_txn(self, txn, pdu_id, origin): + """Get all the signatures for a given PDU. + Args: + txn (cursor): + pdu_id (str): Id for the PDU. + origin (str): origin of the PDU. + Returns: + A dict of key_id -> signature_bytes. + """ + query = ( + "SELECT key_id, signature" + " FROM pdu_origin_signatures" + " WHERE WHERE pdu_id = ? and origin = ?" + ) + txn.execute(query, (pdu_id, origin)) + return dict(txn.fetchall()) + + def _store_pdu_origin_signature_txn(self, txn, pdu_id, origin, key_id, + signature_bytes): + """Store a signature from the origin server for a PDU. + Args: + txn (cursor): + pdu_id (str): Id for the PDU. + origin (str): origin of the PDU. + key_id (str): Id for the signing key. + signature (bytes): The signature. + """ + self._simple_insert_txn(self, txn, "pdu_origin_signatures", { + "pdu_id": pdu_id, + "origin": origin, + "key_id": key_id, + "signature": buffer(signature_bytes), + }) + From 5fefc12d1e2da56895d5652e3d7516ac59ab8824 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Oct 2014 16:59:51 +0100 Subject: [PATCH 002/227] Begin implementing state groups. --- synapse/state.py | 87 +++++++++++++++++++++++++++++++- synapse/storage/__init__.py | 6 ++- synapse/storage/schema/state.sql | 33 ++++++++++++ 3 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 synapse/storage/schema/state.sql diff --git a/synapse/state.py b/synapse/state.py index 9db84c9b5c..8f09b7d50a 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -35,7 +35,7 @@ KeyStateTuple = namedtuple("KeyStateTuple", ("context", "type", "state_key")) class StateHandler(object): - """ Repsonsible for doing state conflict resolution. + """ Responsible for doing state conflict resolution. """ def __init__(self, hs): @@ -50,7 +50,7 @@ class StateHandler(object): to update the state and b) works out what the prev_state should be. Returns: - Deferred: Resolved with a boolean indicating if we succesfully + Deferred: Resolved with a boolean indicating if we successfully updated the state. Raised: @@ -83,6 +83,8 @@ class StateHandler(object): current_state.pdu_id, current_state.origin ) + yield self.update_state_groups(event) + # TODO check current_state to see if the min power level is less # than the power level of the user # power_level = self._get_power_level_for_event(event) @@ -128,6 +130,87 @@ class StateHandler(object): defer.returnValue(is_new) + @defer.inlineCallbacks + def update_state_groups(self, event): + state_groups = yield self.store.get_state_groups( + event.prev_events + ) + + if len(state_groups) == 1 and not hasattr(event, "state_key"): + event.state_group = state_groups[0].group + event.current_state = state_groups[0].state + return + + state = {} + state_sets = {} + for group in state_groups: + for s in group.state: + state.setdefault((s.type, s.state_key), []).add(s) + + state_sets.setdefault( + (s.type, s.state_key), + set() + ).add(s.event_id) + + unconflicted_state = { + k: v.pop() for k, v in state_sets.items() + if len(v) == 1 + } + + conflicted_state = { + k: state[k] + for k, v in state_sets.items() + if len(v) > 1 + } + + new_state = {} + new_state.update(unconflicted_state) + for key, events in conflicted_state.items(): + new_state[key] = yield self.resolve(events) + + if hasattr(event, "state_key"): + new_state[(event.type, event.state_key)] = event + + event.state_group = None + event.current_state = new_state.values() + + @defer.inlineCallbacks + def resolve(self, events): + curr_events = events + + new_powers_deferreds = [] + for e in curr_events: + new_powers_deferreds.append( + self.store.get_power_level(e.context, e.user_id) + ) + + new_powers = yield defer.gatherResults( + new_powers_deferreds, + consumeErrors=True + ) + + max_power = max([int(p) for p in new_powers]) + + curr_events = [ + z[0] for z in zip(curr_events, new_powers) + if int(z[1]) == max_power + ] + + if not curr_events: + raise RuntimeError("Max didn't get a max?") + elif len(curr_events) == 1: + defer.returnValue(curr_events[0]) + + # TODO: For now, just choose the one with the largest event_id. + defer.returnValue( + sorted( + curr_events, + key=lambda e: hashlib.sha1( + e.event_id + e.user_id + e.room_id + e.type + ).hexdigest() + )[0] + ) + def _get_power_level_for_event(self, event): # return self._persistence.get_power_level_for_user(event.room_id, # event.sender) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6dadeb8cce..10456688ef 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -40,6 +40,7 @@ from .stream import StreamStore from .pdu import StatePduStore, PduStore, PdusTable from .transactions import TransactionStore from .keys import KeyStore +from .state import StateStore import json import logging @@ -59,6 +60,7 @@ SCHEMAS = [ "room_aliases", "keys", "redactions", + "state", ] @@ -76,7 +78,7 @@ class _RollbackButIsFineException(Exception): class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, PduStore, StatePduStore, TransactionStore, - DirectoryStore, KeyStore): + DirectoryStore, KeyStore, StateStore): def __init__(self, hs): super(DataStore, self).__init__(hs) @@ -222,6 +224,8 @@ class DataStore(RoomMemberStore, RoomStore, ) raise _RollbackButIsFineException("_persist_event") + self._store_state_groups_txn(txn, event) + is_state = hasattr(event, "state_key") and event.state_key is not None if is_new_state and is_state: vals = { diff --git a/synapse/storage/schema/state.sql b/synapse/storage/schema/state.sql new file mode 100644 index 0000000000..b5c345fae7 --- /dev/null +++ b/synapse/storage/schema/state.sql @@ -0,0 +1,33 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS state_groups( + id INTEGER PRIMARY KEY AUTOINCREMENT, + room_id TEXT NOT NULL, + event_id TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS state_groups_state( + state_group INTEGER NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + event_id TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS event_to_state_groups( + event_id TEXT NOT NULL, + state_group INTEGER NOT NULL +); \ No newline at end of file From 80472ac1988da937f8318651fe5fc764ea869d35 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Oct 2014 10:04:55 +0100 Subject: [PATCH 003/227] Add missing package storate.state --- synapse/storage/state.py | 97 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 synapse/storage/state.py diff --git a/synapse/storage/state.py b/synapse/storage/state.py new file mode 100644 index 0000000000..b910646d74 --- /dev/null +++ b/synapse/storage/state.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import SQLBaseStore +from twisted.internet import defer + +from collections import namedtuple + + +StateGroup = namedtuple("StateGroup", ("group", "state")) + + +class StateStore(SQLBaseStore): + + @defer.inlineCallbacks + def get_state_groups(self, event_ids): + groups = set() + for event_id in event_ids: + group = yield self._simple_select_one_onecol( + table="event_to_state_groups", + keyvalues={"event_id": event_id}, + retcol="state_group", + allow_none=True, + ) + if group: + groups.add(group) + + res = [] + for group in groups: + state_ids = yield self._simple_select_onecol( + table="state_groups_state", + keyvalues={"state_group": group}, + retcol="event_id", + ) + state = [] + for state_id in state_ids: + s = yield self.get_event( + state_id, + allow_none=True, + ) + if s: + state.append(s) + + res.append(StateGroup(group, state)) + + defer.returnValue(res) + + def store_state_groups(self, event): + return self.runInteraction( + self._store_state_groups_txn, event + ) + + def _store_state_groups_txn(self, txn, event): + state_group = event.state_group + if not state_group: + state_group = self._simple_insert_txn( + txn, + table="state_groups", + values={ + "room_id": event.room_id, + "event_id": event.event_id, + } + ) + + for state in event.state_events: + self._simple_insert_txn( + txn, + table="state_groups_state", + values={ + "state_group": state_group, + "room_id": state.room_id, + "type": state.type, + "state_key": state.state_key, + "event_id": state.event_id, + } + ) + + self._simple_insert_txn( + txn, + table="event_to_state_groups", + values={ + "state_group": state_group, + "event_id": event.event_id, + } + ) From e7bc1291a079224315cea5c756061ad711241be1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Oct 2014 16:06:59 +0100 Subject: [PATCH 004/227] Begin making auth use event.old_state_events --- synapse/api/auth.py | 113 +++++++++++++++++++------------ synapse/handlers/_base.py | 10 ++- synapse/handlers/directory.py | 5 +- synapse/handlers/federation.py | 11 ++- synapse/handlers/message.py | 13 +--- synapse/handlers/profile.py | 5 +- synapse/handlers/room.py | 19 +++--- synapse/state.py | 18 ++--- synapse/storage/schema/state.sql | 2 +- synapse/storage/state.py | 2 +- 10 files changed, 115 insertions(+), 83 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index e1b1823cd7..d951cb265b 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,6 +21,7 @@ from synapse.api.constants import Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.api.events.room import ( RoomMemberEvent, RoomPowerLevelsEvent, RoomRedactionEvent, + RoomJoinRulesEvent, RoomOpsPowerLevelsEvent, ) from synapse.util.logutils import log_function @@ -55,11 +56,7 @@ class Auth(object): defer.returnValue(allowed) return - self._check_joined_room( - member=snapshot.membership_state, - user_id=snapshot.user_id, - room_id=snapshot.room_id, - ) + self.check_event_sender_in_room(event) if is_state: # TODO (erikj): This really only should be called for *new* @@ -98,6 +95,16 @@ class Auth(object): pass defer.returnValue(None) + def check_event_sender_in_room(self, event): + key = (RoomMemberEvent.TYPE, event.user_id, ) + member_event = event.state_events.get(key) + + return self._check_joined_room( + member_event, + event.user_id, + event.room_id + ) + def _check_joined_room(self, member, user_id, room_id): if not member or member.membership != Membership.JOIN: raise AuthError(403, "User %s not in room %s (%s)" % ( @@ -114,29 +121,39 @@ class Auth(object): raise AuthError(403, "Room does not exist") # get info about the caller - try: - caller = yield self.store.get_room_member( - user_id=event.user_id, - room_id=event.room_id) - except: - caller = None + key = (RoomMemberEvent.TYPE, event.user_id, ) + caller = event.old_state_events.get(key) + caller_in_room = caller and caller.membership == "join" # get info about the target - try: - target = yield self.store.get_room_member( - user_id=target_user_id, - room_id=event.room_id) - except: - target = None + key = (RoomMemberEvent.TYPE, target_user_id, ) + target = event.old_state_events.get(key) + target_in_room = target and target.membership == "join" membership = event.content["membership"] - join_rule = yield self.store.get_room_join_rule(event.room_id) - if not join_rule: + key = (RoomJoinRulesEvent.TYPE, "", ) + join_rule_event = event.old_state_events.get(key) + if join_rule_event: + join_rule = join_rule_event.content.get( + "join_rule", JoinRules.INVITE + ) + else: join_rule = JoinRules.INVITE + user_level = self._get_power_level_from_event_state( + event, + event.user_id, + ) + + ban_level, kick_level, redact_level = ( + yield self._get_ops_level_from_event_state( + event + ) + ) + if Membership.INVITE == membership: # TODO (erikj): We should probably handle this more intelligently # PRIVATE join rules. @@ -171,29 +188,16 @@ class Auth(object): if not caller_in_room: # trying to leave a room you aren't joined raise AuthError(403, "You are not in room %s." % event.room_id) elif target_user_id != event.user_id: - user_level = yield self.store.get_power_level( - event.room_id, - event.user_id, - ) - _, kick_level, _ = yield self.store.get_ops_levels(event.room_id) - if kick_level: kick_level = int(kick_level) else: - kick_level = 50 + kick_level = 50 # FIXME (erikj): What should we do here? if user_level < kick_level: raise AuthError( 403, "You cannot kick user %s." % target_user_id ) elif Membership.BAN == membership: - user_level = yield self.store.get_power_level( - event.room_id, - event.user_id, - ) - - ban_level, _, _ = yield self.store.get_ops_levels(event.room_id) - if ban_level: ban_level = int(ban_level) else: @@ -206,6 +210,29 @@ class Auth(object): defer.returnValue(True) + def _get_power_level_from_event_state(self, event, user_id): + key = (RoomPowerLevelsEvent.TYPE, "", ) + power_level_event = event.old_state_events.get(key) + level = None + if power_level_event: + level = power_level_event.content[user_id] + if not level: + level = power_level_event.content["default"] + + return level + + def _get_ops_level_from_event_state(self, event): + key = (RoomOpsPowerLevelsEvent.TYPE, "", ) + ops_event = event.old_state_events.get(key) + + if ops_event: + return ( + ops_event.content.get("ban_level"), + ops_event.content.get("kick_level"), + ops_event.content.get("redact_level"), + ) + return None, None, None, + @defer.inlineCallbacks def get_user_by_req(self, request): """ Get a registered user's ID. @@ -282,8 +309,8 @@ class Auth(object): else: send_level = 0 - user_level = yield self.store.get_power_level( - event.room_id, + user_level = self._get_power_level_from_event_state( + event, event.user_id, ) @@ -308,8 +335,8 @@ class Auth(object): add_level = int(add_level) - user_level = yield self.store.get_power_level( - event.room_id, + user_level = self._get_power_level_from_event_state( + event, event.user_id, ) @@ -333,8 +360,8 @@ class Auth(object): if current_state: current_state = current_state[0] - user_level = yield self.store.get_power_level( - event.room_id, + user_level = self._get_power_level_from_event_state( + event, event.user_id, ) @@ -363,10 +390,10 @@ class Auth(object): event.user_id, ) - if user_level: - user_level = int(user_level) - else: - user_level = 0 + user_level = self._get_power_level_from_event_state( + event, + event.user_id, + ) _, _, redact_level = yield self.store.get_ops_levels(event.room_id) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index de4d23bbb3..cd6c35f194 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -44,9 +44,17 @@ class BaseHandler(object): @defer.inlineCallbacks def _on_new_room_event(self, event, snapshot, extra_destinations=[], - extra_users=[]): + extra_users=[], suppress_auth=False): snapshot.fill_out_prev_events(event) + yield self.state_handler.annotate_state_groups(event) + + if not suppress_auth: + yield self.auth.check(event, snapshot, raises=True) + + if hasattr(event, "state_key"): + yield self.state_handler.handle_new_event(event, snapshot) + yield self.store.persist_event(event) destinations = set(extra_destinations) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index a56830d520..6e897e915d 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -152,5 +152,6 @@ class DirectoryHandler(BaseHandler): user_id=user_id, ) - yield self.state_handler.handle_new_event(event, snapshot) - yield self._on_new_room_event(event, snapshot, extra_users=[user_id]) + yield self._on_new_room_event( + event, snapshot, extra_users=[user_id], suppress_auth=True + ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index f52591d2a3..44bf7def2e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -95,6 +95,8 @@ class FederationHandler(BaseHandler): logger.debug("Got event: %s", event.event_id) + yield self.state_handler.annotate_state_groups(event) + with (yield self.lock_manager.lock(pdu.context)): if event.is_state and not backfilled: is_new_state = yield self.state_handler.handle_new_state( @@ -195,7 +197,12 @@ class FederationHandler(BaseHandler): for pdu in pdus: event = self.pdu_codec.event_from_pdu(pdu) + + # FIXME (erikj): Not sure this actually works :/ + yield self.state_handler.annotate_state_groups(event) + events.append(event) + yield self.store.persist_event(event, backfilled=True) defer.returnValue(events) @@ -235,6 +242,7 @@ class FederationHandler(BaseHandler): new_event.destinations = [target_host] snapshot.fill_out_prev_events(new_event) + yield self.state_handler.annotate_state_groups(new_event) yield self.handle_new_event(new_event, snapshot) # TODO (erikj): Time out here. @@ -254,12 +262,11 @@ class FederationHandler(BaseHandler): is_public=False ) except: + # FIXME pass - defer.returnValue(True) - @log_function def _on_user_joined(self, user, room_id): waiters = self.waiting_for_join_list.get((user.to_string(), room_id), []) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 317ef2c80c..1c2cbce151 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -87,10 +87,9 @@ class MessageHandler(BaseHandler): snapshot = yield self.store.snapshot_room(event.room_id, event.user_id) - if not suppress_auth: - yield self.auth.check(event, snapshot, raises=True) - - yield self._on_new_room_event(event, snapshot) + yield self._on_new_room_event( + event, snapshot, suppress_auth=suppress_auth + ) self.hs.get_handlers().presence_handler.bump_presence_active_time( user @@ -149,13 +148,9 @@ class MessageHandler(BaseHandler): state_key=event.state_key, ) - yield self.auth.check(event, snapshot, raises=True) - if stamp_event: event.content["hsob_ts"] = int(self.clock.time_msec()) - yield self.state_handler.handle_new_event(event, snapshot) - yield self._on_new_room_event(event, snapshot) @defer.inlineCallbacks @@ -227,8 +222,6 @@ class MessageHandler(BaseHandler): snapshot = yield self.store.snapshot_room(event.room_id, event.user_id) - yield self.auth.check(event, snapshot, raises=True) - # store message in db yield self._on_new_room_event(event, snapshot) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index dab9b03f04..4cd0a06093 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -218,5 +218,6 @@ class ProfileHandler(BaseHandler): user_id=j.state_key, ) - yield self.state_handler.handle_new_event(new_event, snapshot) - yield self._on_new_room_event(new_event, snapshot) + yield self._on_new_room_event( + new_event, snapshot, suppress_auth=True + ) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c0f9a7c807..cb5bd17d2b 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -129,8 +129,9 @@ class RoomCreationHandler(BaseHandler): logger.debug("Event: %s", event) - yield self.state_handler.handle_new_event(event, snapshot) - yield self._on_new_room_event(event, snapshot, extra_users=[user]) + yield self._on_new_room_event( + event, snapshot, extra_users=[user], suppress_auth=True + ) for event in creation_events: yield handle_event(event) @@ -396,8 +397,6 @@ class RoomMemberHandler(BaseHandler): yield self._do_join(event, snapshot, do_auth=do_auth) else: # This is not a JOIN, so we can handle it normally. - if do_auth: - yield self.auth.check(event, snapshot, raises=True) # If we're banning someone, set a req power level if event.membership == Membership.BAN: @@ -419,6 +418,7 @@ class RoomMemberHandler(BaseHandler): event, membership=event.content["membership"], snapshot=snapshot, + do_auth=do_auth, ) defer.returnValue({"room_id": room_id}) @@ -507,14 +507,11 @@ class RoomMemberHandler(BaseHandler): if not have_joined: logger.debug("Doing normal join") - if do_auth: - yield self.auth.check(event, snapshot, raises=True) - - yield self.state_handler.handle_new_event(event, snapshot) yield self._do_local_membership_update( event, membership=event.content["membership"], snapshot=snapshot, + do_auth=do_auth, ) user = self.hs.parse_userid(event.user_id) @@ -558,7 +555,8 @@ class RoomMemberHandler(BaseHandler): defer.returnValue([r.room_id for r in rooms]) - def _do_local_membership_update(self, event, membership, snapshot): + def _do_local_membership_update(self, event, membership, snapshot, + do_auth): destinations = [] # If we're inviting someone, then we should also send it to that @@ -575,9 +573,10 @@ class RoomMemberHandler(BaseHandler): return self._on_new_room_event( event, snapshot, extra_destinations=destinations, - extra_users=[target_user] + extra_users=[target_user], suppress_auth=(not do_auth), ) + class RoomListHandler(BaseHandler): @defer.inlineCallbacks diff --git a/synapse/state.py b/synapse/state.py index 8f09b7d50a..9be6b716e2 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -71,6 +71,7 @@ class StateHandler(object): # (w.r.t. to power levels) snapshot.fill_out_prev_events(event) + yield self.annotate_state_groups(event) event.prev_events = [ e for e in event.prev_events if e != event.event_id @@ -83,8 +84,6 @@ class StateHandler(object): current_state.pdu_id, current_state.origin ) - yield self.update_state_groups(event) - # TODO check current_state to see if the min power level is less # than the power level of the user # power_level = self._get_power_level_for_event(event) @@ -131,21 +130,16 @@ class StateHandler(object): defer.returnValue(is_new) @defer.inlineCallbacks - def update_state_groups(self, event): + def annotate_state_groups(self, event): state_groups = yield self.store.get_state_groups( event.prev_events ) - if len(state_groups) == 1 and not hasattr(event, "state_key"): - event.state_group = state_groups[0].group - event.current_state = state_groups[0].state - return - state = {} state_sets = {} for group in state_groups: for s in group.state: - state.setdefault((s.type, s.state_key), []).add(s) + state.setdefault((s.type, s.state_key), []).append(s) state_sets.setdefault( (s.type, s.state_key), @@ -153,7 +147,7 @@ class StateHandler(object): ).add(s.event_id) unconflicted_state = { - k: v.pop() for k, v in state_sets.items() + k: state[k].pop() for k, v in state_sets.items() if len(v) == 1 } @@ -168,11 +162,13 @@ class StateHandler(object): for key, events in conflicted_state.items(): new_state[key] = yield self.resolve(events) + event.old_state_events = new_state + if hasattr(event, "state_key"): new_state[(event.type, event.state_key)] = event event.state_group = None - event.current_state = new_state.values() + event.state_events = new_state @defer.inlineCallbacks def resolve(self, events): diff --git a/synapse/storage/schema/state.sql b/synapse/storage/schema/state.sql index b5c345fae7..b44c56b519 100644 --- a/synapse/storage/schema/state.sql +++ b/synapse/storage/schema/state.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS state_groups( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY, room_id TEXT NOT NULL, event_id TEXT NOT NULL ); diff --git a/synapse/storage/state.py b/synapse/storage/state.py index b910646d74..9496c935a7 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -74,7 +74,7 @@ class StateStore(SQLBaseStore): } ) - for state in event.state_events: + for state in event.state_events.values(): self._simple_insert_txn( txn, table="state_groups_state", From 1c445f88f64beabf0bd9bec3950a4a4c0d529e8a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 15 Oct 2014 17:09:04 +0100 Subject: [PATCH 005/227] persist hashes and origin signatures for PDUs --- synapse/api/events/utils.py | 23 ++++++--- synapse/crypto/event_signing.py | 70 +++++++++++++++++++++++++++ synapse/federation/units.py | 17 ++++++- synapse/storage/__init__.py | 21 +++++++- synapse/storage/pdu.py | 11 ++++- synapse/storage/schema/signatures.sql | 4 +- tests/federation/test_federation.py | 4 +- 7 files changed, 135 insertions(+), 15 deletions(-) create mode 100644 synapse/crypto/event_signing.py diff --git a/synapse/api/events/utils.py b/synapse/api/events/utils.py index c3a32be8c1..7fdf45a264 100644 --- a/synapse/api/events/utils.py +++ b/synapse/api/events/utils.py @@ -27,7 +27,14 @@ def prune_event(event): the user has specified, but we do want to keep necessary information like type, state_key etc. """ + return _prune_event_or_pdu(event.type, event) +def prune_pdu(pdu): + """Removes keys that contain unrestricted and non-essential data from a PDU + """ + return _prune_event_or_pdu(pdu.pdu_type, pdu) + +def _prune_event_or_pdu(event_type, event): # Remove all extraneous fields. event.unrecognized_keys = {} @@ -38,25 +45,25 @@ def prune_event(event): if field in event.content: new_content[field] = event.content[field] - if event.type == RoomMemberEvent.TYPE: + if event_type == RoomMemberEvent.TYPE: add_fields("membership") - elif event.type == RoomCreateEvent.TYPE: + elif event_type == RoomCreateEvent.TYPE: add_fields("creator") - elif event.type == RoomJoinRulesEvent.TYPE: + elif event_type == RoomJoinRulesEvent.TYPE: add_fields("join_rule") - elif event.type == RoomPowerLevelsEvent.TYPE: + elif event_type == RoomPowerLevelsEvent.TYPE: # TODO: Actually check these are valid user_ids etc. add_fields("default") for k, v in event.content.items(): if k.startswith("@") and isinstance(v, (int, long)): new_content[k] = v - elif event.type == RoomAddStateLevelEvent.TYPE: + elif event_type == RoomAddStateLevelEvent.TYPE: add_fields("level") - elif event.type == RoomSendEventLevelEvent.TYPE: + elif event_type == RoomSendEventLevelEvent.TYPE: add_fields("level") - elif event.type == RoomOpsPowerLevelsEvent.TYPE: + elif event_type == RoomOpsPowerLevelsEvent.TYPE: add_fields("kick_level", "ban_level", "redact_level") - elif event.type == RoomAliasesEvent.TYPE: + elif event_type == RoomAliasesEvent.TYPE: add_fields("aliases") event.content = new_content diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py new file mode 100644 index 0000000000..6557727e06 --- /dev/null +++ b/synapse/crypto/event_signing.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from synapse.api.events.utils import prune_pdu +from syutil.jsonutil import encode_canonical_json +from syutil.base64util import encode_base64, decode_base64 +from syutil.crypto.jsonsign import sign_json, verify_signed_json + +import hashlib + + +def hash_event_pdu(pdu, hash_algortithm=hashlib.sha256): + hashed = _compute_hash(pdu, hash_algortithm) + hashes[hashed.name] = encode_base64(hashed.digest()) + pdu.hashes = hashes + return pdu + + +def check_event_pdu_hash(pdu, hash_algorithm=hashlib.sha256): + """Check whether the hash for this PDU matches the contents""" + computed_hash = _compute_hash(pdu, hash_algortithm) + if computed_hash.name not in pdu.hashes: + raise Exception("Algorithm %s not in hashes %s" % ( + computed_hash.name, list(pdu.hashes) + )) + message_hash_base64 = hashes[computed_hash.name] + try: + message_hash_bytes = decode_base64(message_hash_base64) + except: + raise Exception("Invalid base64: %s" % (message_hash_base64,)) + return message_hash_bytes == computed_hash.digest() + + +def _compute_hash(pdu, hash_algorithm): + pdu_json = pdu.get_dict() + pdu_json.pop("meta", None) + pdu_json.pop("signatures", None) + hashes = pdu_json.pop("hashes", {}) + pdu_json_bytes = encode_canonical_json(pdu_json) + return hash_algorithm(pdu_json_bytes) + + +def sign_event_pdu(pdu, signature_name, signing_key): + tmp_pdu = Pdu(**pdu.get_dict()) + tmp_pdu = prune_pdu(tmp_pdu) + pdu_json = tmp_pdu.get_dict() + pdu_jdon = sign_json(pdu_json, signature_name, signing_key) + pdu.signatures = pdu_json["signatures"] + return pdu + + +def verify_signed_event_pdu(pdu, signature_name, verify_key): + tmp_pdu = Pdu(**pdu.get_dict()) + tmp_pdu = prune_pdu(tmp_pdu) + pdu_json = tmp_pdu.get_dict() + verify_signed_json(pdu_json, signature_name, verify_key) diff --git a/synapse/federation/units.py b/synapse/federation/units.py index d97aeb698e..3518efb215 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -18,6 +18,7 @@ server protocol. """ from synapse.util.jsonobject import JsonEncodedObject +from syutil.base64util import encode_base64 import logging import json @@ -63,6 +64,8 @@ class Pdu(JsonEncodedObject): "depth", "content", "outlier", + "hashes", + "signatures", "is_state", # Below this are keys valid only for State Pdus. "state_key", "power_level", @@ -91,7 +94,7 @@ class Pdu(JsonEncodedObject): # just leaving it as a dict. (OR DO WE?!) def __init__(self, destinations=[], is_state=False, prev_pdus=[], - outlier=False, **kwargs): + outlier=False, hashes={}, signatures={}, **kwargs): if is_state: for required_key in ["state_key"]: if required_key not in kwargs: @@ -102,6 +105,8 @@ class Pdu(JsonEncodedObject): is_state=is_state, prev_pdus=prev_pdus, outlier=outlier, + hashes=hashes, + signatures=signatures, **kwargs ) @@ -126,6 +131,16 @@ class Pdu(JsonEncodedObject): if "unrecognized_keys" in d and d["unrecognized_keys"]: args.update(json.loads(d["unrecognized_keys"])) + hashes = { + alg: encode_base64(hsh) + for alg, hsh in pdu_tuple.hashes.items() + } + + signatures = { + kid: encode_base64(sig) + for kid, sig in pdu_tuple.signatures.items() + } + return Pdu( prev_pdus=pdu_tuple.prev_pdu_list, **args diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6dadeb8cce..bfeab7d1e8 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -40,6 +40,8 @@ from .stream import StreamStore from .pdu import StatePduStore, PduStore, PdusTable from .transactions import TransactionStore from .keys import KeyStore +from .signatures import SignatureStore + import json import logging @@ -59,6 +61,7 @@ SCHEMAS = [ "room_aliases", "keys", "redactions", + "signatures", ] @@ -76,7 +79,7 @@ class _RollbackButIsFineException(Exception): class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, PduStore, StatePduStore, TransactionStore, - DirectoryStore, KeyStore): + DirectoryStore, KeyStore, SignatureStore): def __init__(self, hs): super(DataStore, self).__init__(hs) @@ -144,6 +147,8 @@ class DataStore(RoomMemberStore, RoomStore, def _persist_event_pdu_txn(self, txn, pdu): cols = dict(pdu.__dict__) unrec_keys = dict(pdu.unrecognized_keys) + del cols["hashes"] + del cols["signatures"] del cols["content"] del cols["prev_pdus"] cols["content_json"] = json.dumps(pdu.content) @@ -157,6 +162,20 @@ class DataStore(RoomMemberStore, RoomStore, logger.debug("Persisting: %s", repr(cols)) + for hash_alg, hash_base64 in pdu.hashes.items(): + hash_bytes = decode_base64(hash_base64) + self._store_pdu_hash_txn( + txn, pdu.pdu_id, pdu.origin, hash_alg, hash_bytes, + ) + + signatures = pdu.sigatures.get(pdu.orgin, {}) + + for key_id, signature_base64 in signatures: + signature_bytes = decode_base64(signature_base64) + self.store_pdu_origin_signatures_txn( + txn, pdu.pdu_id, pdu.origin, key_id, signature_bytes, + ) + if pdu.is_state: self._persist_state_txn(txn, pdu.prev_pdus, cols) else: diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index d70467dcd6..9d624429b7 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -64,6 +64,11 @@ class PduStore(SQLBaseStore): for r in PduEdgesTable.decode_results(txn.fetchall()) ] + hashes = self._get_pdu_hashes_txn(txn, pdu_id, origin) + signatures = self._get_pdu_origin_signatures_txn( + txn, pdu_id, origin + ) + query = ( "SELECT %(fields)s FROM %(pdus)s as p " "LEFT JOIN %(state)s as s " @@ -80,7 +85,9 @@ class PduStore(SQLBaseStore): row = txn.fetchone() if row: - results.append(PduTuple(PduEntry(*row), edges)) + results.append(PduTuple( + PduEntry(*row), edges, hashes, signatures + )) return results @@ -908,7 +915,7 @@ This does not include a prev_pdus key. PduTuple = namedtuple( "PduTuple", - ("pdu_entry", "prev_pdu_list") + ("pdu_entry", "prev_pdu_list", "hashes", "signatures") ) """ This is a tuple of a `PduEntry` and a list of `PduIdTuple` that represent the `prev_pdus` key of a PDU. diff --git a/synapse/storage/schema/signatures.sql b/synapse/storage/schema/signatures.sql index ba3bbb5471..86ee0f2377 100644 --- a/synapse/storage/schema/signatures.sql +++ b/synapse/storage/schema/signatures.sql @@ -28,9 +28,9 @@ CREATE TABLE IF NOT EXISTS pdu_origin_signatures ( origin TEXT, key_id TEXT, signature BLOB, - CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm) + CONSTRAINT uniqueness UNIQUE (pdu_id, origin, key_id) ); CREATE INDEX IF NOT EXISTS pdu_origin_signatures_id ON pdu_origin_signatures ( - pdu_id, origin, + pdu_id, origin ); diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index d86ce83b28..03b2167cf7 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -41,7 +41,7 @@ def make_pdu(prev_pdus=[], **kwargs): } pdu_fields.update(kwargs) - return PduTuple(PduEntry(**pdu_fields), prev_pdus) + return PduTuple(PduEntry(**pdu_fields), prev_pdus, {}, {}) class FederationTestCase(unittest.TestCase): @@ -183,6 +183,8 @@ class FederationTestCase(unittest.TestCase): "is_state": False, "content": {"testing": "content here"}, "depth": 1, + "hashes": {}, + "signatures": {}, }, ] }, From 66104da10c4191aa1e048f2379190574755109e6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 16 Oct 2014 00:09:48 +0100 Subject: [PATCH 006/227] Sign outgoing PDUs. --- synapse/crypto/event_signing.py | 4 ++-- synapse/federation/pdu_codec.py | 6 +++++- synapse/storage/__init__.py | 7 ++++--- synapse/storage/signatures.py | 6 +++--- tests/federation/test_pdu_codec.py | 13 +++++++++--- tests/rest/test_events.py | 7 +++++-- tests/rest/test_profile.py | 8 ++++++-- tests/rest/test_rooms.py | 32 +++++++++++++++++++++++------- tests/utils.py | 3 ++- 9 files changed, 62 insertions(+), 24 deletions(-) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 6557727e06..a115967c0a 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -15,6 +15,7 @@ # limitations under the License. +from synapse.federation.units import Pdu from synapse.api.events.utils import prune_pdu from syutil.jsonutil import encode_canonical_json from syutil.base64util import encode_base64, decode_base64 @@ -25,8 +26,7 @@ import hashlib def hash_event_pdu(pdu, hash_algortithm=hashlib.sha256): hashed = _compute_hash(pdu, hash_algortithm) - hashes[hashed.name] = encode_base64(hashed.digest()) - pdu.hashes = hashes + pdu.hashes[hashed.name] = encode_base64(hashed.digest()) return pdu diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index cef61108dd..bcac5f9ae8 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -14,6 +14,7 @@ # limitations under the License. from .units import Pdu +from synapse.crypto.event_signing import hash_event_pdu, sign_event_pdu import copy @@ -33,6 +34,7 @@ def encode_event_id(pdu_id, origin): class PduCodec(object): def __init__(self, hs): + self.signing_key = hs.config.signing_key[0] self.server_name = hs.hostname self.event_factory = hs.get_event_factory() self.clock = hs.get_clock() @@ -99,4 +101,6 @@ class PduCodec(object): if "ts" not in kwargs: kwargs["ts"] = int(self.clock.time_msec()) - return Pdu(**kwargs) + pdu = Pdu(**kwargs) + pdu = hash_event_pdu(pdu) + return sign_event_pdu(pdu, self.server_name, self.signing_key) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index bfeab7d1e8..b2a3f0b56c 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -42,6 +42,7 @@ from .transactions import TransactionStore from .keys import KeyStore from .signatures import SignatureStore +from syutil.base64util import decode_base64 import json import logging @@ -168,11 +169,11 @@ class DataStore(RoomMemberStore, RoomStore, txn, pdu.pdu_id, pdu.origin, hash_alg, hash_bytes, ) - signatures = pdu.sigatures.get(pdu.orgin, {}) + signatures = pdu.signatures.get(pdu.origin, {}) - for key_id, signature_base64 in signatures: + for key_id, signature_base64 in signatures.items(): signature_bytes = decode_base64(signature_base64) - self.store_pdu_origin_signatures_txn( + self._store_pdu_origin_signature_txn( txn, pdu.pdu_id, pdu.origin, key_id, signature_bytes, ) diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index bb860f09f0..1f0a680500 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -47,7 +47,7 @@ class SignatureStore(SQLBaseStore): algorithm (str): Hashing algorithm. hash_bytes (bytes): Hash function output bytes. """ - self._simple_insert_txn(self, txn, "pdu_hashes", { + self._simple_insert_txn(txn, "pdu_hashes", { "pdu_id": pdu_id, "origin": origin, "algorithm": algorithm, @@ -66,7 +66,7 @@ class SignatureStore(SQLBaseStore): query = ( "SELECT key_id, signature" " FROM pdu_origin_signatures" - " WHERE WHERE pdu_id = ? and origin = ?" + " WHERE pdu_id = ? and origin = ?" ) txn.execute(query, (pdu_id, origin)) return dict(txn.fetchall()) @@ -81,7 +81,7 @@ class SignatureStore(SQLBaseStore): key_id (str): Id for the signing key. signature (bytes): The signature. """ - self._simple_insert_txn(self, txn, "pdu_origin_signatures", { + self._simple_insert_txn(txn, "pdu_origin_signatures", { "pdu_id": pdu_id, "origin": origin, "key_id": key_id, diff --git a/tests/federation/test_pdu_codec.py b/tests/federation/test_pdu_codec.py index 344e1baf60..80851a4258 100644 --- a/tests/federation/test_pdu_codec.py +++ b/tests/federation/test_pdu_codec.py @@ -23,14 +23,21 @@ from synapse.federation.units import Pdu from synapse.server import HomeServer -from mock import Mock +from mock import Mock, NonCallableMock + +from ..utils import MockKey class PduCodecTestCase(unittest.TestCase): def setUp(self): - self.hs = HomeServer("blargle.net") - self.event_factory = self.hs.get_event_factory() + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + self.hs = HomeServer( + "blargle.net", + config=self.mock_config, + ) + self.event_factory = self.hs.get_event_factory() self.codec = PduCodec(self.hs) def test_decode_event_id(self): diff --git a/tests/rest/test_events.py b/tests/rest/test_events.py index 79b371c04d..362c7bc01c 100644 --- a/tests/rest/test_events.py +++ b/tests/rest/test_events.py @@ -28,7 +28,7 @@ from synapse.server import HomeServer # python imports import json -from ..utils import MockHttpResource, MemoryDataStore +from ..utils import MockHttpResource, MemoryDataStore, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -122,6 +122,9 @@ class EventStreamPermissionsTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "test", db_pool=None, @@ -139,7 +142,7 @@ class EventStreamPermissionsTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) diff --git a/tests/rest/test_profile.py b/tests/rest/test_profile.py index b0f48e7fd8..3a0d1e700a 100644 --- a/tests/rest/test_profile.py +++ b/tests/rest/test_profile.py @@ -18,9 +18,9 @@ from tests import unittest from twisted.internet import defer -from mock import Mock +from mock import Mock, NonCallableMock -from ..utils import MockHttpResource +from ..utils import MockHttpResource, MockKey from synapse.api.errors import SynapseError, AuthError from synapse.server import HomeServer @@ -41,6 +41,9 @@ class ProfileTestCase(unittest.TestCase): "set_avatar_url", ]) + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", db_pool=None, http_client=None, @@ -48,6 +51,7 @@ class ProfileTestCase(unittest.TestCase): federation=Mock(), replication_layer=Mock(), datastore=None, + config=self.mock_config, ) def _get_user_by_req(request=None): diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 1ce9b8a83d..7170193051 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -27,7 +27,7 @@ from synapse.server import HomeServer import json import urllib -from ..utils import MockHttpResource, MemoryDataStore +from ..utils import MockHttpResource, MemoryDataStore, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -50,6 +50,9 @@ class RoomPermissionsTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -61,7 +64,7 @@ class RoomPermissionsTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -408,6 +411,9 @@ class RoomsMemberListTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -419,7 +425,7 @@ class RoomsMemberListTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -497,6 +503,9 @@ class RoomsCreateTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -508,7 +517,7 @@ class RoomsCreateTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -598,6 +607,9 @@ class RoomTopicTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -609,7 +621,7 @@ class RoomTopicTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -712,6 +724,9 @@ class RoomMemberStateTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -723,7 +738,7 @@ class RoomMemberStateTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -853,6 +868,9 @@ class RoomMessagesTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -864,7 +882,7 @@ class RoomMessagesTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) diff --git a/tests/utils.py b/tests/utils.py index 60fd6085ac..d8be73dba8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -118,13 +118,14 @@ class MockHttpResource(HttpServer): class MockKey(object): alg = "mock_alg" version = "mock_version" + signature = b"\x9a\x87$" @property def verify_key(self): return self def sign(self, message): - return b"\x9a\x87$" + return self def verify(self, message, sig): assert sig == b"\x9a\x87$" From 1116f5330ec80533954026f67018e0db190cbae0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Oct 2014 16:56:51 +0100 Subject: [PATCH 007/227] Start implementing the invite/join dance. Continue moving auth to use event.state_events --- synapse/api/auth.py | 16 +++--- synapse/federation/replication.py | 22 ++++++-- synapse/federation/transport.py | 34 ++++++++++++- synapse/handlers/federation.py | 83 ++++++++++++++++++++++++++++--- 4 files changed, 133 insertions(+), 22 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index d951cb265b..12ddef1b00 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,7 +21,7 @@ from synapse.api.constants import Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.api.events.room import ( RoomMemberEvent, RoomPowerLevelsEvent, RoomRedactionEvent, - RoomJoinRulesEvent, RoomOpsPowerLevelsEvent, + RoomJoinRulesEvent, RoomOpsPowerLevelsEvent, InviteJoinEvent, ) from synapse.util.logutils import log_function @@ -56,7 +56,8 @@ class Auth(object): defer.returnValue(allowed) return - self.check_event_sender_in_room(event) + if not event.type == InviteJoinEvent.TYPE: + self.check_event_sender_in_room(event) if is_state: # TODO (erikj): This really only should be called for *new* @@ -115,11 +116,6 @@ class Auth(object): def is_membership_change_allowed(self, event): target_user_id = event.state_key - # does this room even exist - room = yield self.store.get_room(event.room_id) - if not room: - raise AuthError(403, "Room does not exist") - # get info about the caller key = (RoomMemberEvent.TYPE, event.user_id, ) caller = event.old_state_events.get(key) @@ -170,7 +166,7 @@ class Auth(object): # joined: It's a NOOP if event.user_id != target_user_id: raise AuthError(403, "Cannot force another user to join.") - elif join_rule == JoinRules.PUBLIC or room.is_public: + elif join_rule == JoinRules.PUBLIC: pass elif join_rule == JoinRules.INVITE: if ( @@ -215,9 +211,9 @@ class Auth(object): power_level_event = event.old_state_events.get(key) level = None if power_level_event: - level = power_level_event.content[user_id] + level = power_level_event.content.get(user_id) if not level: - level = power_level_event.content["default"] + level = power_level_event.content.get("default", 0) return level diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 2346d55045..08c29dece5 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -393,9 +393,25 @@ class ReplicationLayer(object): response = yield self.query_handlers[query_type](args) defer.returnValue((200, response)) else: - defer.returnValue((404, "No handler for Query type '%s'" - % (query_type) - )) + defer.returnValue( + (404, "No handler for Query type '%s'" % (query_type, )) + ) + + def on_make_join_request(self, context, user_id): + return self.handler.on_make_join_request(context, user_id) + + @defer.inlineCallbacks + def on_send_join_request(self, origin, content): + pdu = Pdu(**content) + state = yield self.handler.on_send_join_request(origin, pdu) + defer.returnValue((200, self._transaction_from_pdus(state).get_dict())) + + def make_join(self, destination, context, user_id): + return self.transport_layer.make_join( + destination=destination, + context=context, + user_id=user_id, + ) @defer.inlineCallbacks @log_function diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 755eee8cf6..4f552272e6 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -197,6 +197,19 @@ class TransportLayer(object): defer.returnValue(response) + @defer.inlineCallbacks + @log_function + def make_join(self, destination, context, user_id, retry_on_dns_fail=True): + path = PREFIX + "/make_join/%s/%s" % (context, user_id,) + + response = yield self.client.get_json( + destination=destination, + path=path, + retry_on_dns_fail=retry_on_dns_fail, + ) + + defer.returnValue(response) + @defer.inlineCallbacks def _authenticate_request(self, request): json_request = { @@ -353,6 +366,12 @@ class TransportLayer(object): ) ) + self.server.register_path( + "GET", + re.compile("^" + PREFIX + "/make_join/([^/]*)/([^/]*)$"), + self._on_make_join_request + ) + @defer.inlineCallbacks @log_function def _on_send_request(self, origin, content, query, transaction_id): @@ -438,7 +457,20 @@ class TransportLayer(object): versions = [v.split(",", 1) for v in v_list] return self.request_handler.on_backfill_request( - context, versions, limit) + context, versions, limit + ) + + @log_function + def _on_make_join_request(self, origin, content, query, context, user_id): + return self.request_handler.on_make_join_request( + context, user_id, + ) + + @log_function + def _on_send_join_request(self, origin, content, query): + return self.request_handler.on_send_join_request( + origin, content, + ) class TransportReceivedHandler(object): diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 44bf7def2e..a4f6c739c3 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -89,7 +89,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def on_receive_pdu(self, pdu, backfilled): """ Called by the ReplicationLayer when we have a new pdu. We need to - do auth checks and put it throught the StateHandler. + do auth checks and put it through the StateHandler. """ event = self.pdu_codec.event_from_pdu(pdu) @@ -97,13 +97,17 @@ class FederationHandler(BaseHandler): yield self.state_handler.annotate_state_groups(event) - with (yield self.lock_manager.lock(pdu.context)): - if event.is_state and not backfilled: - is_new_state = yield self.state_handler.handle_new_state( - pdu - ) - else: - is_new_state = False + logger.debug("Event: %s", event) + + if not backfilled: + yield self.auth.check(event, None, raises=True) + + if event.is_state and not backfilled: + is_new_state = yield self.state_handler.handle_new_state( + pdu + ) + else: + is_new_state = False # TODO: Implement something in federation that allows us to # respond to PDU. @@ -267,6 +271,69 @@ class FederationHandler(BaseHandler): defer.returnValue(True) + @defer.inlineCallbacks + def on_make_join_request(self, context, user_id): + event = self.event_factory.create_event( + etype=RoomMemberEvent.TYPE, + content={"membership": Membership.JOIN}, + room_id=context, + user_id=user_id, + state_key=user_id, + ) + + snapshot = yield self.store.snapshot_room( + event.room_id, event.user_id, + ) + snapshot.fill_out_prev_events(event) + + pdu = self.pdu_codec.pdu_from_event(event) + + defer.returnValue(pdu) + + @defer.inlineCallbacks + def on_send_join_request(self, origin, pdu): + event = self.pdu_codec.event_from_pdu(pdu) + + yield self.state_handler.annotate_state_groups(event) + yield self.auth.check(event, None, raises=True) + + is_new_state = yield self.state_handler.handle_new_state( + pdu + ) + + # FIXME (erikj): All this is duplicated above :( + + yield self.store.persist_event( + event, + backfilled=False, + is_new_state=is_new_state + ) + + extra_users = [] + if event.type == RoomMemberEvent.TYPE: + target_user_id = event.state_key + target_user = self.hs.parse_userid(target_user_id) + extra_users.append(target_user) + + yield self.notifier.on_new_room_event( + event, extra_users=extra_users + ) + + if event.type == RoomMemberEvent.TYPE: + if event.membership == Membership.JOIN: + user = self.hs.parse_userid(event.state_key) + self.distributor.fire( + "user_joined_room", user=user, room_id=event.room_id + ) + + pdu.destinations = yield self.store.get_joined_hosts_for_room( + event.room_id + ) + + yield self.replication_layer.send_pdu(pdu) + + defer.returnValue(event.state_events.values()) + @log_function def _on_user_joined(self, user, room_id): waiters = self.waiting_for_join_list.get((user.to_string(), room_id), []) From bb04447c44036ebf3ae5dde7a4cc7a7909d50ef6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 16 Oct 2014 23:25:12 +0100 Subject: [PATCH 008/227] Include hashes of previous pdus when referencing them --- synapse/api/events/__init__.py | 2 +- synapse/federation/pdu_codec.py | 13 ++++------- synapse/federation/replication.py | 2 +- synapse/federation/units.py | 10 ++++++++- synapse/state.py | 4 ---- synapse/storage/__init__.py | 20 +++++++++++------ synapse/storage/pdu.py | 22 +++++++++++++------ synapse/storage/schema/signatures.sql | 16 ++++++++++++++ synapse/storage/signatures.py | 31 +++++++++++++++++++++++++++ tests/federation/test_federation.py | 2 +- tests/federation/test_pdu_codec.py | 4 ++-- 11 files changed, 95 insertions(+), 31 deletions(-) diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index f66fea2904..a5a55742e0 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -65,13 +65,13 @@ class SynapseEvent(JsonEncodedObject): internal_keys = [ "is_state", - "prev_events", "depth", "destinations", "origin", "outlier", "power_level", "redacted", + "prev_pdus", ] required_keys = [ diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index bcac5f9ae8..11fd7264b3 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -45,9 +45,7 @@ class PduCodec(object): kwargs["event_id"] = encode_event_id(pdu.pdu_id, pdu.origin) kwargs["room_id"] = pdu.context kwargs["etype"] = pdu.pdu_type - kwargs["prev_events"] = [ - encode_event_id(p[0], p[1]) for p in pdu.prev_pdus - ] + kwargs["prev_pdus"] = pdu.prev_pdus if hasattr(pdu, "prev_state_id") and hasattr(pdu, "prev_state_origin"): kwargs["prev_state"] = encode_event_id( @@ -78,11 +76,8 @@ class PduCodec(object): d["context"] = event.room_id d["pdu_type"] = event.type - if hasattr(event, "prev_events"): - d["prev_pdus"] = [ - decode_event_id(e, self.server_name) - for e in event.prev_events - ] + if hasattr(event, "prev_pdus"): + d["prev_pdus"] = event.prev_pdus if hasattr(event, "prev_state"): d["prev_state_id"], d["prev_state_origin"] = ( @@ -95,7 +90,7 @@ class PduCodec(object): kwargs = copy.deepcopy(event.unrecognized_keys) kwargs.update({ k: v for k, v in d.items() - if k not in ["event_id", "room_id", "type", "prev_events"] + if k not in ["event_id", "room_id", "type"] }) if "ts" not in kwargs: diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 9363ac7300..788a49b8e8 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -443,7 +443,7 @@ class ReplicationLayer(object): min_depth = yield self.store.get_min_depth_for_context(pdu.context) if min_depth and pdu.depth > min_depth: - for pdu_id, origin in pdu.prev_pdus: + for pdu_id, origin, hashes in pdu.prev_pdus: exists = yield self._get_persisted_pdu(pdu_id, origin) if not exists: diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 3518efb215..6a43007837 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -141,8 +141,16 @@ class Pdu(JsonEncodedObject): for kid, sig in pdu_tuple.signatures.items() } + prev_pdus = [] + for prev_pdu in pdu_tuple.prev_pdu_list: + prev_hashes = pdu_tuple.edge_hashes.get(prev_pdu, {}) + prev_hashes = { + alg: encode_base64(hsh) for alg, hsh in prev_hashes.items() + } + prev_pdus.append((prev_pdu[0], prev_pdu[1], prev_hashes)) + return Pdu( - prev_pdus=pdu_tuple.prev_pdu_list, + prev_pdus=prev_pdus, **args ) else: diff --git a/synapse/state.py b/synapse/state.py index 9db84c9b5c..bc6b928ec7 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -72,10 +72,6 @@ class StateHandler(object): snapshot.fill_out_prev_events(event) - event.prev_events = [ - e for e in event.prev_events if e != event.event_id - ] - current_state = snapshot.prev_state_pdu if current_state: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index b2a3f0b56c..af05b47932 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -177,6 +177,14 @@ class DataStore(RoomMemberStore, RoomStore, txn, pdu.pdu_id, pdu.origin, key_id, signature_bytes, ) + for prev_pdu_id, prev_origin, prev_hashes in pdu.prev_pdus: + for alg, hash_base64 in prev_hashes.items(): + hash_bytes = decode_base64(hash_base64) + self._store_prev_pdu_hash_txn( + txn, pdu.pdu_id, pdu.origin, prev_pdu_id, prev_origin, alg, + hash_bytes + ) + if pdu.is_state: self._persist_state_txn(txn, pdu.prev_pdus, cols) else: @@ -352,6 +360,7 @@ class DataStore(RoomMemberStore, RoomStore, prev_pdus = self._get_latest_pdus_in_context( txn, room_id ) + if state_type is not None and state_key is not None: prev_state_pdu = self._get_current_state_pdu( txn, room_id, state_type, state_key @@ -401,17 +410,16 @@ class Snapshot(object): self.prev_state_pdu = prev_state_pdu def fill_out_prev_events(self, event): - if hasattr(event, "prev_events"): + if hasattr(event, "prev_pdus"): return - es = [ - "%s@%s" % (p_id, origin) for p_id, origin, _ in self.prev_pdus + event.prev_pdus = [ + (p_id, origin, hashes) + for p_id, origin, hashes, _ in self.prev_pdus ] - event.prev_events = [e for e in es if e != event.event_id] - if self.prev_pdus: - event.depth = max([int(v) for _, _, v in self.prev_pdus]) + 1 + event.depth = max([int(v) for _, _, _, v in self.prev_pdus]) + 1 else: event.depth = 0 diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index 9d624429b7..a423b42dbd 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -20,10 +20,13 @@ from ._base import SQLBaseStore, Table, JoinHelper from synapse.federation.units import Pdu from synapse.util.logutils import log_function +from syutil.base64util import encode_base64 + from collections import namedtuple import logging + logger = logging.getLogger(__name__) @@ -64,6 +67,8 @@ class PduStore(SQLBaseStore): for r in PduEdgesTable.decode_results(txn.fetchall()) ] + edge_hashes = self._get_prev_pdu_hashes_txn(txn, pdu_id, origin) + hashes = self._get_pdu_hashes_txn(txn, pdu_id, origin) signatures = self._get_pdu_origin_signatures_txn( txn, pdu_id, origin @@ -86,7 +91,7 @@ class PduStore(SQLBaseStore): row = txn.fetchone() if row: results.append(PduTuple( - PduEntry(*row), edges, hashes, signatures + PduEntry(*row), edges, hashes, signatures, edge_hashes )) return results @@ -310,9 +315,14 @@ class PduStore(SQLBaseStore): (context, ) ) - results = txn.fetchall() + results = [] + for pdu_id, origin, depth in txn.fetchall(): + hashes = self._get_pdu_hashes_txn(txn, pdu_id, origin) + sha256_bytes = hashes["sha256"] + prev_hashes = {"sha256": encode_base64(sha256_bytes)} + results.append((pdu_id, origin, prev_hashes, depth)) - return [(row[0], row[1], row[2]) for row in results] + return results @defer.inlineCallbacks def get_oldest_pdus_in_context(self, context): @@ -431,7 +441,7 @@ class PduStore(SQLBaseStore): "DELETE FROM %s WHERE pdu_id = ? AND origin = ?" % PduForwardExtremitiesTable.table_name ) - txn.executemany(query, prev_pdus) + txn.executemany(query, list(p[:2] for p in prev_pdus)) # We only insert as a forward extremety the new pdu if there are no # other pdus that reference it as a prev pdu @@ -454,7 +464,7 @@ class PduStore(SQLBaseStore): # deleted in a second if they're incorrect anyway. txn.executemany( PduBackwardExtremitiesTable.insert_statement(), - [(i, o, context) for i, o in prev_pdus] + [(i, o, context) for i, o, _ in prev_pdus] ) # Also delete from the backwards extremities table all ones that @@ -915,7 +925,7 @@ This does not include a prev_pdus key. PduTuple = namedtuple( "PduTuple", - ("pdu_entry", "prev_pdu_list", "hashes", "signatures") + ("pdu_entry", "prev_pdu_list", "hashes", "signatures", "edge_hashes") ) """ This is a tuple of a `PduEntry` and a list of `PduIdTuple` that represent the `prev_pdus` key of a PDU. diff --git a/synapse/storage/schema/signatures.sql b/synapse/storage/schema/signatures.sql index 86ee0f2377..a72c4dc35f 100644 --- a/synapse/storage/schema/signatures.sql +++ b/synapse/storage/schema/signatures.sql @@ -34,3 +34,19 @@ CREATE TABLE IF NOT EXISTS pdu_origin_signatures ( CREATE INDEX IF NOT EXISTS pdu_origin_signatures_id ON pdu_origin_signatures ( pdu_id, origin ); + +CREATE TABLE IF NOT EXISTS pdu_edge_hashes( + pdu_id TEXT, + origin TEXT, + prev_pdu_id TEXT, + prev_origin TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE ( + pdu_id, origin, prev_pdu_id, prev_origin, algorithm + ) +); + +CREATE INDEX IF NOT EXISTS pdu_edge_hashes_id ON pdu_edge_hashes( + pdu_id, origin +); diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 1f0a680500..1147102489 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -88,3 +88,34 @@ class SignatureStore(SQLBaseStore): "signature": buffer(signature_bytes), }) + def _get_prev_pdu_hashes_txn(self, txn, pdu_id, origin): + """Get all the hashes for previous PDUs of a PDU + Args: + txn (cursor): + pdu_id (str): Id of the PDU. + origin (str): Origin of the PDU. + Returns: + dict of (pdu_id, origin) -> dict of algorithm -> hash_bytes. + """ + query = ( + "SELECT prev_pdu_id, prev_origin, algorithm, hash" + " FROM pdu_edge_hashes" + " WHERE pdu_id = ? and origin = ?" + ) + txn.execute(query, (pdu_id, origin)) + results = {} + for prev_pdu_id, prev_origin, algorithm, hash_bytes in txn.fetchall(): + hashes = results.setdefault((prev_pdu_id, prev_origin), {}) + hashes[algorithm] = hash_bytes + return results + + def _store_prev_pdu_hash_txn(self, txn, pdu_id, origin, prev_pdu_id, + prev_origin, algorithm, hash_bytes): + self._simple_insert_txn(txn, "pdu_edge_hashes", { + "pdu_id": pdu_id, + "origin": origin, + "prev_pdu_id": prev_pdu_id, + "prev_origin": prev_origin, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }) diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 03b2167cf7..eed50e6335 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -41,7 +41,7 @@ def make_pdu(prev_pdus=[], **kwargs): } pdu_fields.update(kwargs) - return PduTuple(PduEntry(**pdu_fields), prev_pdus, {}, {}) + return PduTuple(PduEntry(**pdu_fields), prev_pdus, {}, {}, {}) class FederationTestCase(unittest.TestCase): diff --git a/tests/federation/test_pdu_codec.py b/tests/federation/test_pdu_codec.py index 80851a4258..0ad8cf6641 100644 --- a/tests/federation/test_pdu_codec.py +++ b/tests/federation/test_pdu_codec.py @@ -88,7 +88,7 @@ class PduCodecTestCase(unittest.TestCase): self.assertEquals(pdu.context, event.room_id) self.assertEquals(pdu.is_state, event.is_state) self.assertEquals(pdu.depth, event.depth) - self.assertEquals(["alice@bob.com"], event.prev_events) + self.assertEquals(pdu.prev_pdus, event.prev_pdus) self.assertEquals(pdu.content, event.content) def test_pdu_from_event(self): @@ -144,7 +144,7 @@ class PduCodecTestCase(unittest.TestCase): self.assertEquals(pdu.context, event.room_id) self.assertEquals(pdu.is_state, event.is_state) self.assertEquals(pdu.depth, event.depth) - self.assertEquals(["alice@bob.com"], event.prev_events) + self.assertEquals(pdu.prev_pdus, event.prev_pdus) self.assertEquals(pdu.content, event.content) self.assertEquals(pdu.state_key, event.state_key) From c8f996e29ffd7055bc6521ea610fc12ff50502e5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Oct 2014 11:40:35 +0100 Subject: [PATCH 009/227] Hash the same content covered by the signature when referencing previous PDUs rather than reusing the PDU content hashes --- synapse/crypto/event_signing.py | 19 +++++++++--- synapse/federation/pdu_codec.py | 6 ++-- synapse/storage/__init__.py | 9 +++++- synapse/storage/pdu.py | 4 +-- synapse/storage/schema/signatures.sql | 18 +++++++++-- synapse/storage/signatures.py | 44 ++++++++++++++++++++++++--- 6 files changed, 84 insertions(+), 16 deletions(-) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index a115967c0a..32d60bd30a 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -24,15 +24,15 @@ from syutil.crypto.jsonsign import sign_json, verify_signed_json import hashlib -def hash_event_pdu(pdu, hash_algortithm=hashlib.sha256): - hashed = _compute_hash(pdu, hash_algortithm) +def add_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): + hashed = _compute_content_hash(pdu, hash_algorithm) pdu.hashes[hashed.name] = encode_base64(hashed.digest()) return pdu -def check_event_pdu_hash(pdu, hash_algorithm=hashlib.sha256): +def check_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): """Check whether the hash for this PDU matches the contents""" - computed_hash = _compute_hash(pdu, hash_algortithm) + computed_hash = _compute_content_hash(pdu, hash_algortithm) if computed_hash.name not in pdu.hashes: raise Exception("Algorithm %s not in hashes %s" % ( computed_hash.name, list(pdu.hashes) @@ -45,7 +45,7 @@ def check_event_pdu_hash(pdu, hash_algorithm=hashlib.sha256): return message_hash_bytes == computed_hash.digest() -def _compute_hash(pdu, hash_algorithm): +def _compute_content_hash(pdu, hash_algorithm): pdu_json = pdu.get_dict() pdu_json.pop("meta", None) pdu_json.pop("signatures", None) @@ -54,6 +54,15 @@ def _compute_hash(pdu, hash_algorithm): return hash_algorithm(pdu_json_bytes) +def compute_pdu_event_reference_hash(pdu, hash_algorithm=hashlib.sha256): + tmp_pdu = Pdu(**pdu.get_dict()) + tmp_pdu = prune_pdu(tmp_pdu) + pdu_json = tmp_pdu.get_dict() + pdu_json_bytes = encode_canonical_json(pdu_json) + hashed = hash_algorithm(pdu_json_bytes) + return (hashed.name, hashed.digest()) + + def sign_event_pdu(pdu, signature_name, signing_key): tmp_pdu = Pdu(**pdu.get_dict()) tmp_pdu = prune_pdu(tmp_pdu) diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index 11fd7264b3..7e574f451d 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -14,7 +14,9 @@ # limitations under the License. from .units import Pdu -from synapse.crypto.event_signing import hash_event_pdu, sign_event_pdu +from synapse.crypto.event_signing import ( + add_event_pdu_content_hash, sign_event_pdu +) import copy @@ -97,5 +99,5 @@ class PduCodec(object): kwargs["ts"] = int(self.clock.time_msec()) pdu = Pdu(**kwargs) - pdu = hash_event_pdu(pdu) + pdu = add_event_pdu_content_hash(pdu) return sign_event_pdu(pdu, self.server_name, self.signing_key) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index af05b47932..1738260cc1 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -44,6 +44,8 @@ from .signatures import SignatureStore from syutil.base64util import decode_base64 +from synapse.crypto.event_signing import compute_pdu_event_reference_hash + import json import logging import os @@ -165,7 +167,7 @@ class DataStore(RoomMemberStore, RoomStore, for hash_alg, hash_base64 in pdu.hashes.items(): hash_bytes = decode_base64(hash_base64) - self._store_pdu_hash_txn( + self._store_pdu_content_hash_txn( txn, pdu.pdu_id, pdu.origin, hash_alg, hash_bytes, ) @@ -185,6 +187,11 @@ class DataStore(RoomMemberStore, RoomStore, hash_bytes ) + (ref_alg, ref_hash_bytes) = compute_pdu_event_reference_hash(pdu) + self._store_pdu_reference_hash_txn( + txn, pdu.pdu_id, pdu.origin, ref_alg, ref_hash_bytes + ) + if pdu.is_state: self._persist_state_txn(txn, pdu.prev_pdus, cols) else: diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index a423b42dbd..3a90c382f0 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -69,7 +69,7 @@ class PduStore(SQLBaseStore): edge_hashes = self._get_prev_pdu_hashes_txn(txn, pdu_id, origin) - hashes = self._get_pdu_hashes_txn(txn, pdu_id, origin) + hashes = self._get_pdu_content_hashes_txn(txn, pdu_id, origin) signatures = self._get_pdu_origin_signatures_txn( txn, pdu_id, origin ) @@ -317,7 +317,7 @@ class PduStore(SQLBaseStore): results = [] for pdu_id, origin, depth in txn.fetchall(): - hashes = self._get_pdu_hashes_txn(txn, pdu_id, origin) + hashes = self._get_pdu_reference_hashes_txn(txn, pdu_id, origin) sha256_bytes = hashes["sha256"] prev_hashes = {"sha256": encode_base64(sha256_bytes)} results.append((pdu_id, origin, prev_hashes, depth)) diff --git a/synapse/storage/schema/signatures.sql b/synapse/storage/schema/signatures.sql index a72c4dc35f..1c45a51bec 100644 --- a/synapse/storage/schema/signatures.sql +++ b/synapse/storage/schema/signatures.sql @@ -13,7 +13,7 @@ * limitations under the License. */ -CREATE TABLE IF NOT EXISTS pdu_hashes ( +CREATE TABLE IF NOT EXISTS pdu_content_hashes ( pdu_id TEXT, origin TEXT, algorithm TEXT, @@ -21,7 +21,21 @@ CREATE TABLE IF NOT EXISTS pdu_hashes ( CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm) ); -CREATE INDEX IF NOT EXISTS pdu_hashes_id ON pdu_hashes (pdu_id, origin); +CREATE INDEX IF NOT EXISTS pdu_content_hashes_id ON pdu_content_hashes ( + pdu_id, origin +); + +CREATE TABLE IF NOT EXISTS pdu_reference_hashes ( + pdu_id TEXT, + origin TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm) +); + +CREATE INDEX IF NOT EXISTS pdu_reference_hashes_id ON pdu_reference_hashes ( + pdu_id, origin +); CREATE TABLE IF NOT EXISTS pdu_origin_signatures ( pdu_id TEXT, diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 1147102489..85eec7ffbe 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -21,7 +21,7 @@ from twisted.internet import defer class SignatureStore(SQLBaseStore): """Persistence for PDU signatures and hashes""" - def _get_pdu_hashes_txn(self, txn, pdu_id, origin): + def _get_pdu_content_hashes_txn(self, txn, pdu_id, origin): """Get all the hashes for a given PDU. Args: txn (cursor): @@ -32,13 +32,14 @@ class SignatureStore(SQLBaseStore): """ query = ( "SELECT algorithm, hash" - " FROM pdu_hashes" + " FROM pdu_content_hashes" " WHERE pdu_id = ? and origin = ?" ) txn.execute(query, (pdu_id, origin)) return dict(txn.fetchall()) - def _store_pdu_hash_txn(self, txn, pdu_id, origin, algorithm, hash_bytes): + def _store_pdu_content_hash_txn(self, txn, pdu_id, origin, algorithm, + hash_bytes): """Store a hash for a PDU Args: txn (cursor): @@ -47,13 +48,48 @@ class SignatureStore(SQLBaseStore): algorithm (str): Hashing algorithm. hash_bytes (bytes): Hash function output bytes. """ - self._simple_insert_txn(txn, "pdu_hashes", { + self._simple_insert_txn(txn, "pdu_content_hashes", { "pdu_id": pdu_id, "origin": origin, "algorithm": algorithm, "hash": buffer(hash_bytes), }) + def _get_pdu_reference_hashes_txn(self, txn, pdu_id, origin): + """Get all the hashes for a given PDU. + Args: + txn (cursor): + pdu_id (str): Id for the PDU. + origin (str): origin of the PDU. + Returns: + A dict of algorithm -> hash. + """ + query = ( + "SELECT algorithm, hash" + " FROM pdu_reference_hashes" + " WHERE pdu_id = ? and origin = ?" + ) + txn.execute(query, (pdu_id, origin)) + return dict(txn.fetchall()) + + def _store_pdu_reference_hash_txn(self, txn, pdu_id, origin, algorithm, + hash_bytes): + """Store a hash for a PDU + Args: + txn (cursor): + pdu_id (str): Id for the PDU. + origin (str): origin of the PDU. + algorithm (str): Hashing algorithm. + hash_bytes (bytes): Hash function output bytes. + """ + self._simple_insert_txn(txn, "pdu_reference_hashes", { + "pdu_id": pdu_id, + "origin": origin, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }) + + def _get_pdu_origin_signatures_txn(self, txn, pdu_id, origin): """Get all the signatures for a given PDU. Args: From f71627567b4aa58c5aba7e79c6d972b8ac26b449 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Oct 2014 15:04:17 +0100 Subject: [PATCH 010/227] Finish implementing the new join dance. --- synapse/api/auth.py | 9 ++ synapse/api/events/factory.py | 14 ++- synapse/federation/replication.py | 66 ++++++++--- synapse/federation/transport.py | 68 +++++++++-- synapse/handlers/federation.py | 187 ++++++++++++++---------------- synapse/state.py | 10 +- 6 files changed, 225 insertions(+), 129 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 12ddef1b00..d1eca791ab 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -48,6 +48,15 @@ class Auth(object): """ try: if hasattr(event, "room_id"): + if not event.old_state_events: + # Oh, we don't know what the state of the room was, so we + # are trusting that this is allowed (at least for now) + defer.returnValue(True) + + if hasattr(event, "outlier") and event.outlier: + # TODO (erikj): Auth for outliers is done differently. + defer.returnValue(True) + is_state = hasattr(event, "state_key") if event.type == RoomMemberEvent.TYPE: diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index 0d94850cec..c6d1151cac 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -51,12 +51,20 @@ class EventFactory(object): self.clock = hs.get_clock() self.hs = hs + self.event_id_count = 0 + + def create_event_id(self): + i = str(self.event_id_count) + self.event_id_count += 1 + + local_part = str(int(self.clock.time())) + i + random_string(5) + + return "%s@%s" % (local_part, self.hs.hostname) + def create_event(self, etype=None, **kwargs): kwargs["type"] = etype if "event_id" not in kwargs: - kwargs["event_id"] = "%s@%s" % ( - random_string(10), self.hs.hostname - ) + kwargs["event_id"] = self.create_event_id() if "ts" not in kwargs: kwargs["ts"] = int(self.clock.time_msec()) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 08c29dece5..d482193851 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -244,13 +244,14 @@ class ReplicationLayer(object): pdu = None if pdu_list: pdu = pdu_list[0] - yield self._handle_new_pdu(pdu) + yield self._handle_new_pdu(destination, pdu) defer.returnValue(pdu) @defer.inlineCallbacks @log_function - def get_state_for_context(self, destination, context): + def get_state_for_context(self, destination, context, pdu_id=None, + pdu_origin=None): """Requests all of the `current` state PDUs for a given context from a remote home server. @@ -263,13 +264,14 @@ class ReplicationLayer(object): """ transaction_data = yield self.transport_layer.get_context_state( - destination, context) + destination, context, pdu_id=pdu_id, pdu_origin=pdu_origin, + ) transaction = Transaction(**transaction_data) pdus = [Pdu(outlier=True, **p) for p in transaction.pdus] for pdu in pdus: - yield self._handle_new_pdu(pdu) + yield self._handle_new_pdu(destination, pdu) defer.returnValue(pdus) @@ -315,7 +317,7 @@ class ReplicationLayer(object): dl = [] for pdu in pdu_list: - dl.append(self._handle_new_pdu(pdu)) + dl.append(self._handle_new_pdu(transaction.origin, pdu)) if hasattr(transaction, "edus"): for edu in [Edu(**x) for x in transaction.edus]: @@ -347,14 +349,19 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def on_context_state_request(self, context): - results = yield self.store.get_current_state_for_context( - context - ) + def on_context_state_request(self, context, pdu_id, pdu_origin): + if pdu_id and pdu_origin: + pdus = yield self.handler.get_state_for_pdu( + pdu_id, pdu_origin + ) + else: + results = yield self.store.get_current_state_for_context( + context + ) + pdus = [Pdu.from_pdu_tuple(p) for p in results] - logger.debug("Context returning %d results", len(results)) + logger.debug("Context returning %d results", len(pdus)) - pdus = [Pdu.from_pdu_tuple(p) for p in results] defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) @defer.inlineCallbacks @@ -396,9 +403,10 @@ class ReplicationLayer(object): defer.returnValue( (404, "No handler for Query type '%s'" % (query_type, )) ) - + @defer.inlineCallbacks def on_make_join_request(self, context, user_id): - return self.handler.on_make_join_request(context, user_id) + pdu = yield self.handler.on_make_join_request(context, user_id) + defer.returnValue(pdu.get_dict()) @defer.inlineCallbacks def on_send_join_request(self, origin, content): @@ -406,13 +414,27 @@ class ReplicationLayer(object): state = yield self.handler.on_send_join_request(origin, pdu) defer.returnValue((200, self._transaction_from_pdus(state).get_dict())) + @defer.inlineCallbacks def make_join(self, destination, context, user_id): - return self.transport_layer.make_join( + pdu_dict = yield self.transport_layer.make_join( destination=destination, context=context, user_id=user_id, ) + logger.debug("Got response to make_join: %s", pdu_dict) + + defer.returnValue(Pdu(**pdu_dict)) + + def send_join(self, destination, pdu): + return self.transport_layer.send_join( + destination, + pdu.context, + pdu.pdu_id, + pdu.origin, + pdu.get_dict(), + ) + @defer.inlineCallbacks @log_function def _get_persisted_pdu(self, pdu_id, pdu_origin): @@ -443,7 +465,7 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def _handle_new_pdu(self, pdu, backfilled=False): + def _handle_new_pdu(self, origin, pdu, backfilled=False): # We reprocess pdus when we have seen them only as outliers existing = yield self._get_persisted_pdu(pdu.pdu_id, pdu.origin) @@ -452,6 +474,8 @@ class ReplicationLayer(object): defer.returnValue({}) return + state = None + # Get missing pdus if necessary. is_new = yield self.pdu_actions.is_new(pdu) if is_new and not pdu.outlier: @@ -475,12 +499,22 @@ class ReplicationLayer(object): except: # TODO(erikj): Do some more intelligent retries. logger.exception("Failed to get PDU") + else: + # We need to get the state at this event, since we have reached + # a backward extremity edge. + state = yield self.get_state_for_context( + origin, pdu.context, pdu.pdu_id, pdu.origin, + ) # Persist the Pdu, but don't mark it as processed yet. yield self.store.persist_event(pdu=pdu) if not backfilled: - ret = yield self.handler.on_receive_pdu(pdu, backfilled=backfilled) + ret = yield self.handler.on_receive_pdu( + pdu, + backfilled=backfilled, + state=state, + ) else: ret = None diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 4f552272e6..a0d34fd24d 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -72,7 +72,8 @@ class TransportLayer(object): self.received_handler = None @log_function - def get_context_state(self, destination, context): + def get_context_state(self, destination, context, pdu_id=None, + pdu_origin=None): """ Requests all state for a given context (i.e. room) from the given server. @@ -89,7 +90,14 @@ class TransportLayer(object): subpath = "/state/%s/" % context - return self._do_request_for_transaction(destination, subpath) + args = {} + if pdu_id and pdu_origin: + args["pdu_id"] = pdu_id + args["pdu_origin"] = pdu_origin + + return self._do_request_for_transaction( + destination, subpath, args=args + ) @log_function def get_pdu(self, destination, pdu_origin, pdu_id): @@ -135,8 +143,10 @@ class TransportLayer(object): subpath = "/backfill/%s/" % context - args = {"v": ["%s,%s" % (i, o) for i, o in pdu_tuples]} - args["limit"] = limit + args = { + "v": ["%s,%s" % (i, o) for i, o in pdu_tuples], + "limit": limit, + } return self._do_request_for_transaction( dest, @@ -210,6 +220,23 @@ class TransportLayer(object): defer.returnValue(response) + @defer.inlineCallbacks + @log_function + def send_join(self, destination, context, pdu_id, origin, content): + path = PREFIX + "/send_join/%s/%s/%s" % ( + context, + origin, + pdu_id, + ) + + response = yield self.client.put_json( + destination=destination, + path=path, + data=content, + ) + + defer.returnValue(response) + @defer.inlineCallbacks def _authenticate_request(self, request): json_request = { @@ -330,7 +357,11 @@ class TransportLayer(object): re.compile("^" + PREFIX + "/state/([^/]*)/$"), self._with_authentication( lambda origin, content, query, context: - handler.on_context_state_request(context) + handler.on_context_state_request( + context, + query.get("pdu_id", [None])[0], + query.get("pdu_origin", [None])[0] + ) ) ) @@ -369,7 +400,23 @@ class TransportLayer(object): self.server.register_path( "GET", re.compile("^" + PREFIX + "/make_join/([^/]*)/([^/]*)$"), - self._on_make_join_request + self._with_authentication( + lambda origin, content, query, context, user_id: + self._on_make_join_request( + origin, content, query, context, user_id + ) + ) + ) + + self.server.register_path( + "PUT", + re.compile("^" + PREFIX + "/send_join/([^/]*)/([^/]*)/([^/]*)$"), + self._with_authentication( + lambda origin, content, query, context, pdu_origin, pdu_id: + self._on_send_join_request( + origin, content, query, + ) + ) ) @defer.inlineCallbacks @@ -460,18 +507,23 @@ class TransportLayer(object): context, versions, limit ) + @defer.inlineCallbacks @log_function def _on_make_join_request(self, origin, content, query, context, user_id): - return self.request_handler.on_make_join_request( + content = yield self.request_handler.on_make_join_request( context, user_id, ) + defer.returnValue((200, content)) + @defer.inlineCallbacks @log_function def _on_send_join_request(self, origin, content, query): - return self.request_handler.on_send_join_request( + content = yield self.request_handler.on_send_join_request( origin, content, ) + defer.returnValue((200, content)) + class TransportReceivedHandler(object): """ Callbacks used when we receive a transaction diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index a4f6c739c3..0ae0541bd3 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -20,7 +20,7 @@ from ._base import BaseHandler from synapse.api.events.room import InviteJoinEvent, RoomMemberEvent from synapse.api.constants import Membership from synapse.util.logutils import log_function -from synapse.federation.pdu_codec import PduCodec +from synapse.federation.pdu_codec import PduCodec, encode_event_id from synapse.api.errors import SynapseError from twisted.internet import defer, reactor @@ -87,7 +87,7 @@ class FederationHandler(BaseHandler): @log_function @defer.inlineCallbacks - def on_receive_pdu(self, pdu, backfilled): + def on_receive_pdu(self, pdu, backfilled, state=None): """ Called by the ReplicationLayer when we have a new pdu. We need to do auth checks and put it through the StateHandler. """ @@ -95,7 +95,10 @@ class FederationHandler(BaseHandler): logger.debug("Got event: %s", event.event_id) - yield self.state_handler.annotate_state_groups(event) + if state: + state = [self.pdu_codec.event_from_pdu(p) for p in state] + state = {(e.type, e.state_key): e for e in state} + yield self.state_handler.annotate_state_groups(event, state=state) logger.debug("Event: %s", event) @@ -108,83 +111,55 @@ class FederationHandler(BaseHandler): ) else: is_new_state = False + # TODO: Implement something in federation that allows us to # respond to PDU. - target_is_mine = False - if hasattr(event, "target_host"): - target_is_mine = event.target_host == self.hs.hostname - - if event.type == InviteJoinEvent.TYPE: - if not target_is_mine: - logger.debug("Ignoring invite/join event %s", event) - return - - # If we receive an invite/join event then we need to join the - # sender to the given room. - # TODO: We should probably auth this or some such - content = event.content - content.update({"membership": Membership.JOIN}) - new_event = self.event_factory.create_event( - etype=RoomMemberEvent.TYPE, - state_key=event.user_id, - room_id=event.room_id, - user_id=event.user_id, - membership=Membership.JOIN, - content=content + with (yield self.room_lock.lock(event.room_id)): + yield self.store.persist_event( + event, + backfilled, + is_new_state=is_new_state ) - yield self.hs.get_handlers().room_member_handler.change_membership( - new_event, - do_auth=False, + room = yield self.store.get_room(event.room_id) + + if not room: + # Huh, let's try and get the current state + try: + yield self.replication_layer.get_state_for_context( + event.origin, event.room_id, pdu.pdu_id, pdu.origin, + ) + + hosts = yield self.store.get_joined_hosts_for_room( + event.room_id + ) + if self.hs.hostname in hosts: + try: + yield self.store.store_room( + room_id=event.room_id, + room_creator_user_id="", + is_public=False, + ) + except: + pass + except: + logger.exception( + "Failed to get current state for room %s", + event.room_id + ) + + if not backfilled: + extra_users = [] + if event.type == RoomMemberEvent.TYPE: + target_user_id = event.state_key + target_user = self.hs.parse_userid(target_user_id) + extra_users.append(target_user) + + yield self.notifier.on_new_room_event( + event, extra_users=extra_users ) - else: - with (yield self.room_lock.lock(event.room_id)): - yield self.store.persist_event( - event, - backfilled, - is_new_state=is_new_state - ) - - room = yield self.store.get_room(event.room_id) - - if not room: - # Huh, let's try and get the current state - try: - yield self.replication_layer.get_state_for_context( - event.origin, event.room_id - ) - - hosts = yield self.store.get_joined_hosts_for_room( - event.room_id - ) - if self.hs.hostname in hosts: - try: - yield self.store.store_room( - room_id=event.room_id, - room_creator_user_id="", - is_public=False, - ) - except: - pass - except: - logger.exception( - "Failed to get current state for room %s", - event.room_id - ) - - if not backfilled: - extra_users = [] - if event.type == RoomMemberEvent.TYPE: - target_user_id = event.state_key - target_user = self.hs.parse_userid(target_user_id) - extra_users.append(target_user) - - yield self.notifier.on_new_room_event( - event, extra_users=extra_users - ) - if event.type == RoomMemberEvent.TYPE: if event.membership == Membership.JOIN: user = self.hs.parse_userid(event.state_key) @@ -214,40 +189,35 @@ class FederationHandler(BaseHandler): @log_function @defer.inlineCallbacks def do_invite_join(self, target_host, room_id, joinee, content, snapshot): - hosts = yield self.store.get_joined_hosts_for_room(room_id) if self.hs.hostname in hosts: # We are already in the room. logger.debug("We're already in the room apparently") defer.returnValue(False) - # First get current state to see if we are already joined. - try: - yield self.replication_layer.get_state_for_context( - target_host, room_id - ) - - hosts = yield self.store.get_joined_hosts_for_room(room_id) - if self.hs.hostname in hosts: - # Oh, we were actually in the room already. - logger.debug("We're already in the room apparently") - defer.returnValue(False) - except Exception: - logger.exception("Failed to get current state") - - new_event = self.event_factory.create_event( - etype=InviteJoinEvent.TYPE, - target_host=target_host, - room_id=room_id, - user_id=joinee, - content=content + pdu = yield self.replication_layer.make_join( + target_host, + room_id, + joinee ) - new_event.destinations = [target_host] + logger.debug("Got response to make_join: %s", pdu) - snapshot.fill_out_prev_events(new_event) - yield self.state_handler.annotate_state_groups(new_event) - yield self.handle_new_event(new_event, snapshot) + event = self.pdu_codec.event_from_pdu(pdu) + + # We should assert some things. + assert(event.type == RoomMemberEvent.TYPE) + assert(event.user_id == joinee) + assert(event.state_key == joinee) + assert(event.room_id == room_id) + + event.event_id = self.event_factory.create_event_id() + event.content = content + + state = yield self.replication_layer.send_join( + target_host, + self.pdu_codec.pdu_from_event(event) + ) # TODO (erikj): Time out here. d = defer.Deferred() @@ -326,14 +296,31 @@ class FederationHandler(BaseHandler): "user_joined_room", user=user, room_id=event.room_id ) - pdu.destinations = yield self.store.get_joined_hosts_for_room( + new_pdu = self.pdu_codec.pdu_from_event(event); + new_pdu.destinations = yield self.store.get_joined_hosts_for_room( event.room_id ) - yield self.replication_layer.send_pdu(pdu) + yield self.replication_layer.send_pdu(new_pdu) defer.returnValue(event.state_events.values()) + @defer.inlineCallbacks + def get_state_for_pdu(self, pdu_id, pdu_origin): + state_groups = yield self.store.get_state_groups( + [encode_event_id(pdu_id, pdu_origin)] + ) + + if state_groups: + defer.returnValue( + [ + self.pdu_codec.pdu_from_event(s) + for s in state_groups[0].state + ] + ) + else: + defer.returnValue([]) + @log_function def _on_user_joined(self, user, room_id): waiters = self.waiting_for_join_list.get((user.to_string(), room_id), []) diff --git a/synapse/state.py b/synapse/state.py index 9be6b716e2..8c4eeb8924 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -130,7 +130,13 @@ class StateHandler(object): defer.returnValue(is_new) @defer.inlineCallbacks - def annotate_state_groups(self, event): + def annotate_state_groups(self, event, state=None): + if state: + event.state_group = None + event.old_state_events = None + event.state_events = state + return + state_groups = yield self.store.get_state_groups( event.prev_events ) @@ -177,7 +183,7 @@ class StateHandler(object): new_powers_deferreds = [] for e in curr_events: new_powers_deferreds.append( - self.store.get_power_level(e.context, e.user_id) + self.store.get_power_level(e.room_id, e.user_id) ) new_powers = yield defer.gatherResults( From 4d1a7624f444deee4352645fbf73165e11f66dd0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Oct 2014 15:27:11 +0100 Subject: [PATCH 011/227] move 'age' into 'meta' subdict so that it is clearer that it is not part of the signed data --- synapse/federation/replication.py | 20 ++++++++++++++------ synapse/federation/units.py | 6 +++++- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 788a49b8e8..c4993aa5ee 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -295,6 +295,10 @@ class ReplicationLayer(object): transaction = Transaction(**transaction_data) for p in transaction.pdus: + if "meta" in p: + meta = p["meta"] + if "age" in meta: + p["age"] = meta["age"] if "age" in p: p["age_ts"] = int(self._clock.time_msec()) - int(p["age"]) del p["age"] @@ -414,14 +418,16 @@ class ReplicationLayer(object): transmission. """ pdus = [p.get_dict() for p in pdu_list] + time_now = self._clock.time_msec() for p in pdus: - if "age_ts" in pdus: - p["age"] = int(self.clock.time_msec()) - p["age_ts"] - + if "age_ts" in p: + age = time_now - p["age_ts"] + p.setdefault("meta", {})["age"] = int(age) + del p["age_ts"] return Transaction( origin=self.server_name, pdus=pdus, - ts=int(self._clock.time_msec()), + ts=int(time_now), destination=None, ) @@ -589,7 +595,7 @@ class _TransactionQueue(object): logger.debug("TX [%s] Persisting transaction...", destination) transaction = Transaction.create_new( - ts=self._clock.time_msec(), + ts=int(self._clock.time_msec()), transaction_id=str(self._next_txn_id), origin=self.server_name, destination=destination, @@ -614,7 +620,9 @@ class _TransactionQueue(object): if "pdus" in data: for p in data["pdus"]: if "age_ts" in p: - p["age"] = now - int(p["age_ts"]) + meta = p.setdefault("meta", {}) + meta["age"] = now - int(p["age_ts"]) + del p["age_ts"] return data code, response = yield self.transport_layer.send_transaction( diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 6a43007837..c4a10a4123 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -68,11 +68,11 @@ class Pdu(JsonEncodedObject): "signatures", "is_state", # Below this are keys valid only for State Pdus. "state_key", - "power_level", "prev_state_id", "prev_state_origin", "required_power_level", "user_id", + "meta" ] internal_keys = [ @@ -124,6 +124,10 @@ class Pdu(JsonEncodedObject): if pdu_tuple: d = copy.copy(pdu_tuple.pdu_entry._asdict()) + for k in d.keys(): + if d[k] is None: + del d[k] + d["content"] = json.loads(d["content_json"]) del d["content_json"] From c5cec1cc77029c21f0117c318c522ab320de3923 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Oct 2014 16:50:04 +0100 Subject: [PATCH 012/227] Rename 'meta' to 'unsigned' --- docs/server-server/signing.rst | 16 ++++++++-------- synapse/crypto/event_signing.py | 4 +++- synapse/federation/replication.py | 14 +++++++------- synapse/federation/units.py | 1 - 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/docs/server-server/signing.rst b/docs/server-server/signing.rst index dae10f121b..60c701ca91 100644 --- a/docs/server-server/signing.rst +++ b/docs/server-server/signing.rst @@ -1,13 +1,13 @@ Signing JSON ============ -JSON is signed by encoding the JSON object without ``signatures`` or ``meta`` +JSON is signed by encoding the JSON object without ``signatures`` or ``unsigned`` keys using a canonical encoding. The JSON bytes are then signed using the signature algorithm and the signature encoded using base64 with the padding stripped. The resulting base64 signature is added to an object under the *signing key identifier* which is added to the ``signatures`` object under the name of the server signing it which is added back to the original JSON object -along with the ``meta`` object. +along with the ``unsigned`` object. The *signing key identifier* is the concatenation of the *signing algorithm* and a *key version*. The *signing algorithm* identifies the algorithm used to @@ -15,8 +15,8 @@ sign the JSON. The currently support value for *signing algorithm* is ``ed25519`` as implemented by NACL (http://nacl.cr.yp.to/). The *key version* is used to distinguish between different signing keys used by the same entity. -The ``meta`` object and the ``signatures`` object are not covered by the -signature. Therefore intermediate servers can add metadata such as time stamps +The ``unsigned`` object and the ``signatures`` object are not covered by the +signature. Therefore intermediate servers can add unsigneddata such as time stamps and additional signatures. @@ -27,7 +27,7 @@ and additional signatures. "signing_keys": { "ed25519:1": "XSl0kuyvrXNj6A+7/tkrB9sxSbRi08Of5uRhxOqZtEQ" }, - "meta": { + "unsigned": { "retrieved_ts_ms": 922834800000 }, "signatures": { @@ -41,7 +41,7 @@ and additional signatures. def sign_json(json_object, signing_key, signing_name): signatures = json_object.pop("signatures", {}) - meta = json_object.pop("meta", None) + unsigned = json_object.pop("unsigned", None) signed = signing_key.sign(encode_canonical_json(json_object)) signature_base64 = encode_base64(signed.signature) @@ -50,8 +50,8 @@ and additional signatures. signatures.setdefault(sigature_name, {})[key_id] = signature_base64 json_object["signatures"] = signatures - if meta is not None: - json_object["meta"] = meta + if unsigned is not None: + json_object["unsigned"] = unsigned return json_object diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 32d60bd30a..a236f7d708 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -47,7 +47,9 @@ def check_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): def _compute_content_hash(pdu, hash_algorithm): pdu_json = pdu.get_dict() - pdu_json.pop("meta", None) + #TODO: Make "age_ts" key internal + pdu_json.pop("age_ts") + pdu_json.pop("unsigned", None) pdu_json.pop("signatures", None) hashes = pdu_json.pop("hashes", {}) pdu_json_bytes = encode_canonical_json(pdu_json) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index c4993aa5ee..f2a5d4d5e2 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -295,10 +295,10 @@ class ReplicationLayer(object): transaction = Transaction(**transaction_data) for p in transaction.pdus: - if "meta" in p: - meta = p["meta"] - if "age" in meta: - p["age"] = meta["age"] + if "unsigned" in p: + unsigned = p["unsigned"] + if "age" in unsigned: + p["age"] = unsigned["age"] if "age" in p: p["age_ts"] = int(self._clock.time_msec()) - int(p["age"]) del p["age"] @@ -422,7 +422,7 @@ class ReplicationLayer(object): for p in pdus: if "age_ts" in p: age = time_now - p["age_ts"] - p.setdefault("meta", {})["age"] = int(age) + p.setdefault("unsigned", {})["age"] = int(age) del p["age_ts"] return Transaction( origin=self.server_name, @@ -620,8 +620,8 @@ class _TransactionQueue(object): if "pdus" in data: for p in data["pdus"]: if "age_ts" in p: - meta = p.setdefault("meta", {}) - meta["age"] = now - int(p["age_ts"]) + unsigned = p.setdefault("unsigned", {}) + unsigned["age"] = now - int(p["age_ts"]) del p["age_ts"] return data diff --git a/synapse/federation/units.py b/synapse/federation/units.py index c4a10a4123..c629e5793e 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -72,7 +72,6 @@ class Pdu(JsonEncodedObject): "prev_state_origin", "required_power_level", "user_id", - "meta" ] internal_keys = [ From 5ffe5ab43fa090111a0141b04ce6342172f60724 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Oct 2014 18:56:42 +0100 Subject: [PATCH 013/227] Use state groups to get current state. Make join dance actually work. --- synapse/api/auth.py | 5 ++ synapse/federation/replication.py | 17 +++++- synapse/federation/transport.py | 57 ++++++++++++++++-- synapse/handlers/federation.py | 74 ++++++++++++++++------- synapse/handlers/message.py | 6 +- synapse/rest/base.py | 5 ++ synapse/rest/events.py | 32 ++++++---- synapse/state.py | 98 +++++++++++++++++++++---------- synapse/storage/pdu.py | 6 ++ synapse/storage/state.py | 3 + 10 files changed, 231 insertions(+), 72 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index d1eca791ab..50ce7eb4cd 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -22,6 +22,7 @@ from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.api.events.room import ( RoomMemberEvent, RoomPowerLevelsEvent, RoomRedactionEvent, RoomJoinRulesEvent, RoomOpsPowerLevelsEvent, InviteJoinEvent, + RoomCreateEvent, ) from synapse.util.logutils import log_function @@ -59,6 +60,10 @@ class Auth(object): is_state = hasattr(event, "state_key") + if event.type == RoomCreateEvent.TYPE: + # FIXME + defer.returnValue(True) + if event.type == RoomMemberEvent.TYPE: yield self._can_replace_state(event) allowed = yield self.is_membership_change_allowed(event) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index d482193851..8c7d510ef6 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -403,11 +403,18 @@ class ReplicationLayer(object): defer.returnValue( (404, "No handler for Query type '%s'" % (query_type, )) ) + @defer.inlineCallbacks def on_make_join_request(self, context, user_id): pdu = yield self.handler.on_make_join_request(context, user_id) defer.returnValue(pdu.get_dict()) + @defer.inlineCallbacks + def on_invite_request(self, origin, content): + pdu = Pdu(**content) + ret_pdu = yield self.handler.on_send_join_request(origin, pdu) + defer.returnValue((200, ret_pdu.get_dict())) + @defer.inlineCallbacks def on_send_join_request(self, origin, content): pdu = Pdu(**content) @@ -426,8 +433,9 @@ class ReplicationLayer(object): defer.returnValue(Pdu(**pdu_dict)) + @defer.inlineCallbacks def send_join(self, destination, pdu): - return self.transport_layer.send_join( + _, content = yield self.transport_layer.send_join( destination, pdu.context, pdu.pdu_id, @@ -435,6 +443,13 @@ class ReplicationLayer(object): pdu.get_dict(), ) + logger.debug("Got content: %s", content) + pdus = [Pdu(outlier=True, **p) for p in content.get("pdus", [])] + for pdu in pdus: + yield self._handle_new_pdu(destination, pdu) + + defer.returnValue(pdus) + @defer.inlineCallbacks @log_function def _get_persisted_pdu(self, pdu_id, pdu_origin): diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index a0d34fd24d..de64702e2f 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -229,13 +229,36 @@ class TransportLayer(object): pdu_id, ) - response = yield self.client.put_json( + code, content = yield self.client.put_json( destination=destination, path=path, data=content, ) - defer.returnValue(response) + if not 200 <= code < 300: + raise RuntimeError("Got %d from send_join", code) + + defer.returnValue(json.loads(content)) + + @defer.inlineCallbacks + @log_function + def send_invite(self, destination, context, pdu_id, origin, content): + path = PREFIX + "/invite/%s/%s/%s" % ( + context, + origin, + pdu_id, + ) + + code, content = yield self.client.put_json( + destination=destination, + path=path, + data=content, + ) + + if not 200 <= code < 300: + raise RuntimeError("Got %d from send_invite", code) + + defer.returnValue(json.loads(content)) @defer.inlineCallbacks def _authenticate_request(self, request): @@ -297,9 +320,13 @@ class TransportLayer(object): @defer.inlineCallbacks def new_handler(request, *args, **kwargs): (origin, content) = yield self._authenticate_request(request) - response = yield handler( - origin, content, request.args, *args, **kwargs - ) + try: + response = yield handler( + origin, content, request.args, *args, **kwargs + ) + except: + logger.exception("Callback failed") + raise defer.returnValue(response) return new_handler @@ -419,6 +446,17 @@ class TransportLayer(object): ) ) + self.server.register_path( + "PUT", + re.compile("^" + PREFIX + "/invite/([^/]*)/([^/]*)/([^/]*)$"), + self._with_authentication( + lambda origin, content, query, context, pdu_origin, pdu_id: + self._on_invite_request( + origin, content, query, + ) + ) + ) + @defer.inlineCallbacks @log_function def _on_send_request(self, origin, content, query, transaction_id): @@ -524,6 +562,15 @@ class TransportLayer(object): defer.returnValue((200, content)) + @defer.inlineCallbacks + @log_function + def _on_invite_request(self, origin, content, query): + content = yield self.request_handler.on_invite_request( + origin, content, + ) + + defer.returnValue((200, content)) + class TransportReceivedHandler(object): """ Callbacks used when we receive a transaction diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0ae0541bd3..70790aaa72 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -62,6 +62,9 @@ class FederationHandler(BaseHandler): self.pdu_codec = PduCodec(hs) + # When joining a room we need to queue any events for that room up + self.room_queues = {} + @log_function @defer.inlineCallbacks def handle_new_event(self, event, snapshot): @@ -95,22 +98,25 @@ class FederationHandler(BaseHandler): logger.debug("Got event: %s", event.event_id) + if event.room_id in self.room_queues: + self.room_queues[event.room_id].append(pdu) + return + if state: state = [self.pdu_codec.event_from_pdu(p) for p in state] state = {(e.type, e.state_key): e for e in state} - yield self.state_handler.annotate_state_groups(event, state=state) + + is_new_state = yield self.state_handler.annotate_state_groups( + event, + state=state + ) logger.debug("Event: %s", event) if not backfilled: yield self.auth.check(event, None, raises=True) - if event.is_state and not backfilled: - is_new_state = yield self.state_handler.handle_new_state( - pdu - ) - else: - is_new_state = False + is_new_state = is_new_state and not backfilled # TODO: Implement something in federation that allows us to # respond to PDU. @@ -211,6 +217,8 @@ class FederationHandler(BaseHandler): assert(event.state_key == joinee) assert(event.room_id == room_id) + self.room_queues[room_id] = [] + event.event_id = self.event_factory.create_event_id() event.content = content @@ -219,15 +227,14 @@ class FederationHandler(BaseHandler): self.pdu_codec.pdu_from_event(event) ) - # TODO (erikj): Time out here. - d = defer.Deferred() - self.waiting_for_join_list.setdefault((joinee, room_id), []).append(d) - reactor.callLater(10, d.cancel) + state = [self.pdu_codec.event_from_pdu(p) for p in state] - try: - yield d - except defer.CancelledError: - raise SynapseError(500, "Unable to join remote room") + logger.debug("do_invite_join state: %s", state) + + is_new_state = yield self.state_handler.annotate_state_groups( + event, + state=state + ) try: yield self.store.store_room( @@ -239,6 +246,32 @@ class FederationHandler(BaseHandler): # FIXME pass + for e in state: + # FIXME: Auth these. + is_new_state = yield self.state_handler.annotate_state_groups( + e, + state=state + ) + + yield self.store.persist_event( + e, + backfilled=False, + is_new_state=False + ) + + yield self.store.persist_event( + event, + backfilled=False, + is_new_state=is_new_state + ) + + room_queue = self.room_queues[room_id] + del self.room_queues[room_id] + + for p in room_queue: + p.outlier = True + yield self.on_receive_pdu(p, backfilled=False) + defer.returnValue(True) @defer.inlineCallbacks @@ -264,13 +297,9 @@ class FederationHandler(BaseHandler): def on_send_join_request(self, origin, pdu): event = self.pdu_codec.event_from_pdu(pdu) - yield self.state_handler.annotate_state_groups(event) + is_new_state= yield self.state_handler.annotate_state_groups(event) yield self.auth.check(event, None, raises=True) - is_new_state = yield self.state_handler.handle_new_state( - pdu - ) - # FIXME (erikj): All this is duplicated above :( yield self.store.persist_event( @@ -303,7 +332,10 @@ class FederationHandler(BaseHandler): yield self.replication_layer.send_pdu(new_pdu) - defer.returnValue(event.state_events.values()) + defer.returnValue([ + self.pdu_codec.pdu_from_event(e) + for e in event.state_events.values() + ]) @defer.inlineCallbacks def get_state_for_pdu(self, pdu_id, pdu_origin): diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 1c2cbce151..4aaf97a83e 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -199,7 +199,7 @@ class MessageHandler(BaseHandler): raise RoomError( 403, "Member does not meet private room rules.") - data = yield self.store.get_current_state( + data = yield self.state_handler.get_current_state( room_id, event_type, state_key ) defer.returnValue(data) @@ -238,7 +238,7 @@ class MessageHandler(BaseHandler): yield self.auth.check_joined_room(room_id, user_id) # TODO: This is duplicating logic from snapshot_all_rooms - current_state = yield self.store.get_current_state(room_id) + current_state = yield self.state_handler.get_current_state(room_id) defer.returnValue([self.hs.serialize_event(c) for c in current_state]) @defer.inlineCallbacks @@ -315,7 +315,7 @@ class MessageHandler(BaseHandler): "end": end_token.to_string(), } - current_state = yield self.store.get_current_state( + current_state = yield self.state_handler.get_current_state( event.room_id ) d["state"] = [self.hs.serialize_event(c) for c in current_state] diff --git a/synapse/rest/base.py b/synapse/rest/base.py index 2e8e3fa7d4..dc784c1527 100644 --- a/synapse/rest/base.py +++ b/synapse/rest/base.py @@ -18,6 +18,11 @@ from synapse.api.urls import CLIENT_PREFIX from synapse.rest.transactions import HttpTransactionStore import re +import logging + + +logger = logging.getLogger(__name__) + def client_path_pattern(path_regex): """Creates a regex compiled client path with the correct client path diff --git a/synapse/rest/events.py b/synapse/rest/events.py index 097195d7cc..92ff5e5ca7 100644 --- a/synapse/rest/events.py +++ b/synapse/rest/events.py @@ -20,6 +20,12 @@ from synapse.api.errors import SynapseError from synapse.streams.config import PaginationConfig from synapse.rest.base import RestServlet, client_path_pattern +import logging + + +logger = logging.getLogger(__name__) + + class EventStreamRestServlet(RestServlet): PATTERN = client_path_pattern("/events$") @@ -29,18 +35,22 @@ class EventStreamRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request): auth_user = yield self.auth.get_user_by_req(request) + try: + handler = self.handlers.event_stream_handler + pagin_config = PaginationConfig.from_request(request) + timeout = EventStreamRestServlet.DEFAULT_LONGPOLL_TIME_MS + if "timeout" in request.args: + try: + timeout = int(request.args["timeout"][0]) + except ValueError: + raise SynapseError(400, "timeout must be in milliseconds.") - handler = self.handlers.event_stream_handler - pagin_config = PaginationConfig.from_request(request) - timeout = EventStreamRestServlet.DEFAULT_LONGPOLL_TIME_MS - if "timeout" in request.args: - try: - timeout = int(request.args["timeout"][0]) - except ValueError: - raise SynapseError(400, "timeout must be in milliseconds.") - - chunk = yield handler.get_stream(auth_user.to_string(), pagin_config, - timeout=timeout) + chunk = yield handler.get_stream( + auth_user.to_string(), pagin_config, timeout=timeout + ) + except: + logger.exception("Event stream failed") + raise defer.returnValue((200, chunk)) diff --git a/synapse/state.py b/synapse/state.py index 8c4eeb8924..24685c6fb4 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.federation.pdu_codec import encode_event_id, decode_event_id from synapse.util.logutils import log_function +from synapse.federation.pdu_codec import encode_event_id from collections import namedtuple @@ -130,43 +131,23 @@ class StateHandler(object): defer.returnValue(is_new) @defer.inlineCallbacks + @log_function def annotate_state_groups(self, event, state=None): if state: event.state_group = None event.old_state_events = None - event.state_events = state + event.state_events = {(s.type, s.state_key): s for s in state} + defer.returnValue(False) return - state_groups = yield self.store.get_state_groups( - event.prev_events - ) + if hasattr(event, "outlier") and event.outlier: + event.state_group = None + event.old_state_events = None + event.state_events = None + defer.returnValue(False) + return - state = {} - state_sets = {} - for group in state_groups: - for s in group.state: - state.setdefault((s.type, s.state_key), []).append(s) - - state_sets.setdefault( - (s.type, s.state_key), - set() - ).add(s.event_id) - - unconflicted_state = { - k: state[k].pop() for k, v in state_sets.items() - if len(v) == 1 - } - - conflicted_state = { - k: state[k] - for k, v in state_sets.items() - if len(v) > 1 - } - - new_state = {} - new_state.update(unconflicted_state) - for key, events in conflicted_state.items(): - new_state[key] = yield self.resolve(events) + new_state = yield self.resolve_state_groups(event.prev_events) event.old_state_events = new_state @@ -176,8 +157,63 @@ class StateHandler(object): event.state_group = None event.state_events = new_state + defer.returnValue(hasattr(event, "state_key")) + @defer.inlineCallbacks - def resolve(self, events): + def get_current_state(self, room_id, event_type=None, state_key=""): + # FIXME: HACK! + pdus = yield self.store.get_latest_pdus_in_context(room_id) + + event_ids = [encode_event_id(p.pdu_id, p.origin) for p in pdus] + + res = self.resolve_state_groups(event_ids) + + if event_type: + defer.returnValue(res.get((event_type, state_key))) + return + + defer.returnValue(res.values()) + + @defer.inlineCallbacks + @log_function + def resolve_state_groups(self, event_ids): + state_groups = yield self.store.get_state_groups( + event_ids + ) + + state = {} + for group in state_groups: + for s in group.state: + state.setdefault( + (s.type, s.state_key), + {} + )[s.event_id] = s + + unconflicted_state = { + k: v.values()[0] for k, v in state.items() + if len(v.values()) == 1 + } + + conflicted_state = { + k: v.values() + for k, v in state.items() + if len(v.values()) > 1 + } + + try: + new_state = {} + new_state.update(unconflicted_state) + for key, events in conflicted_state.items(): + new_state[key] = yield self._resolve_state_events(events) + except: + logger.exception("Failed to resolve state") + raise + + defer.returnValue(new_state) + + @defer.inlineCallbacks + @log_function + def _resolve_state_events(self, events): curr_events = events new_powers_deferreds = [] diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index d70467dcd6..b1cb0185a6 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -277,6 +277,12 @@ class PduStore(SQLBaseStore): (context, depth) ) + def get_latest_pdus_in_context(self, context): + return self.runInteraction( + self._get_latest_pdus_in_context, + context + ) + def _get_latest_pdus_in_context(self, txn, context): """Get's a list of the most current pdus for a given context. This is used when we are sending a Pdu and need to fill out the `prev_pdus` diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 9496c935a7..0aa979c9f0 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -63,6 +63,9 @@ class StateStore(SQLBaseStore): ) def _store_state_groups_txn(self, txn, event): + if not event.state_events: + return + state_group = event.state_group if not state_group: state_group = self._simple_insert_txn( From b3b19614962d78e7851299ff1f7b41706ced3d00 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Oct 2014 19:37:41 +0100 Subject: [PATCH 014/227] Fix bug where people could join private rooms --- synapse/api/auth.py | 86 +++++++++++++++++++--------------- synapse/handlers/federation.py | 10 +++- synapse/state.py | 12 +++-- 3 files changed, 63 insertions(+), 45 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 50ce7eb4cd..93a3533304 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -49,12 +49,12 @@ class Auth(object): """ try: if hasattr(event, "room_id"): - if not event.old_state_events: + if event.old_state_events is None: # Oh, we don't know what the state of the room was, so we # are trusting that this is allowed (at least for now) defer.returnValue(True) - if hasattr(event, "outlier") and event.outlier: + if hasattr(event, "outlier") and event.outlier is True: # TODO (erikj): Auth for outliers is done differently. defer.returnValue(True) @@ -65,8 +65,12 @@ class Auth(object): defer.returnValue(True) if event.type == RoomMemberEvent.TYPE: - yield self._can_replace_state(event) - allowed = yield self.is_membership_change_allowed(event) + self._can_replace_state(event) + allowed = self.is_membership_change_allowed(event) + if allowed: + logger.debug("Allowing! %s", event) + else: + logger.debug("Denying! %s", event) defer.returnValue(allowed) return @@ -77,24 +81,28 @@ class Auth(object): # TODO (erikj): This really only should be called for *new* # state yield self._can_add_state(event) - yield self._can_replace_state(event) + self._can_replace_state(event) else: yield self._can_send_event(event) if event.type == RoomPowerLevelsEvent.TYPE: - yield self._check_power_levels(event) + self._check_power_levels(event) if event.type == RoomRedactionEvent.TYPE: - yield self._check_redaction(event) + self._check_redaction(event) + + logger.debug("Allowing! %s", event) defer.returnValue(True) else: raise AuthError(500, "Unknown event: %s" % event) except AuthError as e: logger.info("Event auth check failed on event %s with msg: %s", event, e.msg) + logger.info("Denying! %s", event) if raises: raise e + defer.returnValue(False) @defer.inlineCallbacks @@ -126,7 +134,7 @@ class Auth(object): user_id, room_id, repr(member) )) - @defer.inlineCallbacks + @log_function def is_membership_change_allowed(self, event): target_user_id = event.state_key @@ -159,11 +167,23 @@ class Auth(object): ) ban_level, kick_level, redact_level = ( - yield self._get_ops_level_from_event_state( + self._get_ops_level_from_event_state( event ) ) + logger.debug( + "is_membership_change_allowed: %s", + { + "caller_in_room": caller_in_room, + "target_in_room": target_in_room, + "membership": membership, + "join_rule": join_rule, + "target_user_id": target_user_id, + "event.user_id": event.user_id, + } + ) + if Membership.INVITE == membership: # TODO (erikj): We should probably handle this more intelligently # PRIVATE join rules. @@ -183,10 +203,7 @@ class Auth(object): elif join_rule == JoinRules.PUBLIC: pass elif join_rule == JoinRules.INVITE: - if ( - not caller or caller.membership not in - [Membership.INVITE, Membership.JOIN] - ): + if not caller_in_room: raise AuthError(403, "You are not invited to this room.") else: # TODO (erikj): may_join list @@ -218,7 +235,7 @@ class Auth(object): else: raise AuthError(500, "Unknown membership %s" % membership) - defer.returnValue(True) + return True def _get_power_level_from_event_state(self, event, user_id): key = (RoomPowerLevelsEvent.TYPE, "", ) @@ -359,17 +376,7 @@ class Auth(object): defer.returnValue(True) - @defer.inlineCallbacks def _can_replace_state(self, event): - current_state = yield self.store.get_current_state( - event.room_id, - event.type, - event.state_key, - ) - - if current_state: - current_state = current_state[0] - user_level = self._get_power_level_from_event_state( event, event.user_id, @@ -383,6 +390,10 @@ class Auth(object): logger.debug( "Checking power level for %s, %s", event.user_id, user_level ) + + key = (event.type, event.state_key, ) + current_state = event.old_state_events.get(key) + if current_state and hasattr(current_state, "required_power_level"): req = current_state.required_power_level @@ -393,19 +404,20 @@ class Auth(object): "You don't have permission to change that state" ) - @defer.inlineCallbacks def _check_redaction(self, event): - user_level = yield self.store.get_power_level( - event.room_id, - event.user_id, - ) - user_level = self._get_power_level_from_event_state( event, event.user_id, ) - _, _, redact_level = yield self.store.get_ops_levels(event.room_id) + if user_level: + user_level = int(user_level) + else: + user_level = 0 + + _, _, redact_level = self.store._get_ops_level_from_event_state( + event.room_id + ) if not redact_level: redact_level = 50 @@ -416,7 +428,6 @@ class Auth(object): "You don't have permission to redact events" ) - @defer.inlineCallbacks def _check_power_levels(self, event): for k, v in event.content.items(): if k == "default": @@ -436,19 +447,16 @@ class Auth(object): except: raise SynapseError(400, "Not a valid power level: %s" % (v,)) - current_state = yield self.store.get_current_state( - event.room_id, - event.type, - event.state_key, - ) + key = (event.type, event.state_key, ) + current_state = event.old_state_events.get(key) if not current_state: return else: current_state = current_state[0] - user_level = yield self.store.get_power_level( - event.room_id, + user_level = self._get_power_level_from_event_state( + event, event.user_id, ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 70790aaa72..8c80a37164 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -269,12 +269,12 @@ class FederationHandler(BaseHandler): del self.room_queues[room_id] for p in room_queue: - p.outlier = True yield self.on_receive_pdu(p, backfilled=False) defer.returnValue(True) @defer.inlineCallbacks + @log_function def on_make_join_request(self, context, user_id): event = self.event_factory.create_event( etype=RoomMemberEvent.TYPE, @@ -289,15 +289,21 @@ class FederationHandler(BaseHandler): ) snapshot.fill_out_prev_events(event) + yield self.state_handler.annotate_state_groups(event) + yield self.auth.check(event, None, raises=True) + pdu = self.pdu_codec.pdu_from_event(event) defer.returnValue(pdu) @defer.inlineCallbacks + @log_function def on_send_join_request(self, origin, pdu): event = self.pdu_codec.event_from_pdu(pdu) - is_new_state= yield self.state_handler.annotate_state_groups(event) + event.outlier = False + + is_new_state = yield self.state_handler.annotate_state_groups(event) yield self.auth.check(event, None, raises=True) # FIXME (erikj): All this is duplicated above :( diff --git a/synapse/state.py b/synapse/state.py index 24685c6fb4..c062cef757 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -22,6 +22,7 @@ from synapse.federation.pdu_codec import encode_event_id from collections import namedtuple +import copy import logging import hashlib @@ -143,13 +144,13 @@ class StateHandler(object): if hasattr(event, "outlier") and event.outlier: event.state_group = None event.old_state_events = None - event.state_events = None + event.state_events = {} defer.returnValue(False) return new_state = yield self.resolve_state_groups(event.prev_events) - event.old_state_events = new_state + event.old_state_events = copy.deepcopy(new_state) if hasattr(event, "state_key"): new_state[(event.type, event.state_key)] = event @@ -164,9 +165,12 @@ class StateHandler(object): # FIXME: HACK! pdus = yield self.store.get_latest_pdus_in_context(room_id) - event_ids = [encode_event_id(p.pdu_id, p.origin) for p in pdus] + event_ids = [ + encode_event_id(pdu_id, origin) + for pdu_id, origin, _ in pdus + ] - res = self.resolve_state_groups(event_ids) + res = yield self.resolve_state_groups(event_ids) if event_type: defer.returnValue(res.get((event_type, state_key))) From 8afbece68319728e20c3b32c2f949fd1745d405e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Oct 2014 19:41:32 +0100 Subject: [PATCH 015/227] Remove signatures from pdu when computing hashes to use for prev pdus, make sure is_state is a boolean. --- synapse/crypto/event_signing.py | 6 +++++- synapse/federation/units.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index a236f7d708..d3b501c6e7 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -22,6 +22,9 @@ from syutil.base64util import encode_base64, decode_base64 from syutil.crypto.jsonsign import sign_json, verify_signed_json import hashlib +import logging + +logger = logging.getLogger(__name__) def add_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): @@ -48,7 +51,7 @@ def check_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): def _compute_content_hash(pdu, hash_algorithm): pdu_json = pdu.get_dict() #TODO: Make "age_ts" key internal - pdu_json.pop("age_ts") + pdu_json.pop("age_ts", None) pdu_json.pop("unsigned", None) pdu_json.pop("signatures", None) hashes = pdu_json.pop("hashes", {}) @@ -60,6 +63,7 @@ def compute_pdu_event_reference_hash(pdu, hash_algorithm=hashlib.sha256): tmp_pdu = Pdu(**pdu.get_dict()) tmp_pdu = prune_pdu(tmp_pdu) pdu_json = tmp_pdu.get_dict() + pdu_json.pop("signatures", None) pdu_json_bytes = encode_canonical_json(pdu_json) hashed = hash_algorithm(pdu_json_bytes) return (hashed.name, hashed.digest()) diff --git a/synapse/federation/units.py b/synapse/federation/units.py index b81e162512..b779d259bd 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -101,7 +101,7 @@ class Pdu(JsonEncodedObject): super(Pdu, self).__init__( destinations=destinations, - is_state=is_state, + is_state=bool(is_state), prev_pdus=prev_pdus, outlier=outlier, hashes=hashes, From bf8cdda2f598acce5a8694fe6482fc929c53715b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Oct 2014 20:10:34 +0100 Subject: [PATCH 016/227] It doesn't want a dict --- synapse/handlers/federation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 8c80a37164..b575986fc3 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -104,7 +104,6 @@ class FederationHandler(BaseHandler): if state: state = [self.pdu_codec.event_from_pdu(p) for p in state] - state = {(e.type, e.state_key): e for e in state} is_new_state = yield self.state_handler.annotate_state_groups( event, @@ -250,7 +249,6 @@ class FederationHandler(BaseHandler): # FIXME: Auth these. is_new_state = yield self.state_handler.annotate_state_groups( e, - state=state ) yield self.store.persist_event( From eea3a29699aba75a02fefe18d0eeba119a845b04 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Oct 2014 20:36:04 +0100 Subject: [PATCH 017/227] Add script to hash exisitng history --- scripts/hash_history.py | 69 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 scripts/hash_history.py diff --git a/scripts/hash_history.py b/scripts/hash_history.py new file mode 100644 index 0000000000..bdad530af8 --- /dev/null +++ b/scripts/hash_history.py @@ -0,0 +1,69 @@ +from synapse.storage.pdu import PduStore +from synapse.storage.signatures import SignatureStore +from synapse.storage._base import SQLBaseStore +from synapse.federation.units import Pdu +from synapse.crypto.event_signing import ( + add_event_pdu_content_hash, compute_pdu_event_reference_hash +) +from synapse.api.events.utils import prune_pdu +from syutil.base64util import encode_base64, decode_base64 +from syutil.jsonutil import encode_canonical_json +import sqlite3 +import sys + +class Store(object): + _get_pdu_tuples = PduStore.__dict__["_get_pdu_tuples"] + _get_pdu_content_hashes_txn = SignatureStore.__dict__["_get_pdu_content_hashes_txn"] + _get_prev_pdu_hashes_txn = SignatureStore.__dict__["_get_prev_pdu_hashes_txn"] + _get_pdu_origin_signatures_txn = SignatureStore.__dict__["_get_pdu_origin_signatures_txn"] + _store_pdu_content_hash_txn = SignatureStore.__dict__["_store_pdu_content_hash_txn"] + _store_pdu_reference_hash_txn = SignatureStore.__dict__["_store_pdu_reference_hash_txn"] + _store_prev_pdu_hash_txn = SignatureStore.__dict__["_store_prev_pdu_hash_txn"] + _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"] + + +store = Store() + + +def select_pdus(cursor): + cursor.execute( + "SELECT pdu_id, origin FROM pdus ORDER BY depth ASC" + ) + + ids = cursor.fetchall() + + pdu_tuples = store._get_pdu_tuples(cursor, ids) + + pdus = [Pdu.from_pdu_tuple(p) for p in pdu_tuples] + + reference_hashes = {} + + for pdu in pdus: + try: + if pdu.prev_pdus: + print "PROCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus + for pdu_id, origin, hashes in pdu.prev_pdus: + ref_alg, ref_hsh = reference_hashes[(pdu_id, origin)] + hashes[ref_alg] = encode_base64(ref_hsh) + store._store_prev_pdu_hash_txn(cursor, pdu.pdu_id, pdu.origin, pdu_id, origin, ref_alg, ref_hsh) + print "SUCCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus + pdu = add_event_pdu_content_hash(pdu) + ref_alg, ref_hsh = compute_pdu_event_reference_hash(pdu) + reference_hashes[(pdu.pdu_id, pdu.origin)] = (ref_alg, ref_hsh) + store._store_pdu_reference_hash_txn(cursor, pdu.pdu_id, pdu.origin, ref_alg, ref_hsh) + + for alg, hsh_base64 in pdu.hashes.items(): + print alg, hsh_base64 + store._store_pdu_content_hash_txn(cursor, pdu.pdu_id, pdu.origin, alg, decode_base64(hsh_base64)) + + except: + print "FAILED_", pdu.pdu_id, pdu.origin, pdu.prev_pdus + +def main(): + conn = sqlite3.connect(sys.argv[1]) + cursor = conn.cursor() + select_pdus(cursor) + conn.commit() + +if __name__=='__main__': + main() From 5e2236f9ffe3a66bbe0ff37b1793e8fa59a1c475 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 27 Oct 2014 11:19:15 +0000 Subject: [PATCH 018/227] fix pyflakes warnings --- synapse/crypto/event_signing.py | 8 ++++---- synapse/federation/units.py | 2 ++ synapse/storage/signatures.py | 2 -- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index d3b501c6e7..61edd2c6f9 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -35,12 +35,12 @@ def add_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): def check_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): """Check whether the hash for this PDU matches the contents""" - computed_hash = _compute_content_hash(pdu, hash_algortithm) + computed_hash = _compute_content_hash(pdu, hash_algorithm) if computed_hash.name not in pdu.hashes: raise Exception("Algorithm %s not in hashes %s" % ( computed_hash.name, list(pdu.hashes) )) - message_hash_base64 = hashes[computed_hash.name] + message_hash_base64 = pdu.hashes[computed_hash.name] try: message_hash_bytes = decode_base64(message_hash_base64) except: @@ -54,7 +54,7 @@ def _compute_content_hash(pdu, hash_algorithm): pdu_json.pop("age_ts", None) pdu_json.pop("unsigned", None) pdu_json.pop("signatures", None) - hashes = pdu_json.pop("hashes", {}) + pdu_json.pop("hashes", None) pdu_json_bytes = encode_canonical_json(pdu_json) return hash_algorithm(pdu_json_bytes) @@ -73,7 +73,7 @@ def sign_event_pdu(pdu, signature_name, signing_key): tmp_pdu = Pdu(**pdu.get_dict()) tmp_pdu = prune_pdu(tmp_pdu) pdu_json = tmp_pdu.get_dict() - pdu_jdon = sign_json(pdu_json, signature_name, signing_key) + pdu_json = sign_json(pdu_json, signature_name, signing_key) pdu.signatures = pdu_json["signatures"] return pdu diff --git a/synapse/federation/units.py b/synapse/federation/units.py index b779d259bd..adc3385644 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -155,6 +155,8 @@ class Pdu(JsonEncodedObject): return Pdu( prev_pdus=prev_pdus, + hashes=hashes, + signatures=signatures, **args ) else: diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 85eec7ffbe..82be946d3f 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -15,8 +15,6 @@ from _base import SQLBaseStore -from twisted.internet import defer - class SignatureStore(SQLBaseStore): """Persistence for PDU signatures and hashes""" From c372929ab6246370d3da8e864f36fcbe2e48123b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Oct 2014 16:31:39 +0000 Subject: [PATCH 019/227] Remove duplicate import --- synapse/state.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/state.py b/synapse/state.py index ec98667c5d..cc6a7db96b 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -18,7 +18,6 @@ from twisted.internet import defer from synapse.federation.pdu_codec import encode_event_id, decode_event_id from synapse.util.logutils import log_function -from synapse.federation.pdu_codec import encode_event_id from collections import namedtuple From 8e358ef35a8e2640176003cdb2396a9b4d71fbce Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Oct 2014 10:34:05 +0000 Subject: [PATCH 020/227] Add timer to LoggingTransaction --- synapse/storage/_base.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 65a86e9056..e839704a8c 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -23,6 +23,7 @@ from synapse.util.logutils import log_function import collections import copy import json +import time logger = logging.getLogger(__name__) @@ -61,9 +62,15 @@ class LoggingTransaction(object): # TODO(paul): Here would be an excellent place to put some timing # measurements, and log (warning?) slow queries. - return object.__getattribute__(self, "txn").execute( - sql, *args, **kwargs - ) + + start = time.clock() * 1000 + try: + return object.__getattribute__(self, "txn").execute( + sql, *args, **kwargs + ) + finally: + end = time.clock() * 1000 + sql_logger.debug("[SQL time] %f", end - start) class SQLBaseStore(object): From 967ce43b59e90f36c21f3a493bc81ab4e57dae69 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Oct 2014 10:53:11 +0000 Subject: [PATCH 021/227] Clean up LoggingTransaction --- synapse/storage/_base.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index e839704a8c..d3e8741889 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -39,14 +39,11 @@ class LoggingTransaction(object): def __init__(self, txn): object.__setattr__(self, "txn", txn) - def __getattribute__(self, name): - if name == "execute": - return object.__getattribute__(self, "execute") - - return getattr(object.__getattribute__(self, "txn"), name) + def __getattr__(self, name): + return getattr(self.txn, name) def __setattr__(self, name, value): - setattr(object.__getattribute__(self, "txn"), name, value) + setattr(self.txn, name, value) def execute(self, sql, *args, **kwargs): # TODO(paul): Maybe use 'info' and 'debug' for values? @@ -60,12 +57,9 @@ class LoggingTransaction(object): # Don't let logging failures stop SQL from working pass - # TODO(paul): Here would be an excellent place to put some timing - # measurements, and log (warning?) slow queries. - start = time.clock() * 1000 try: - return object.__getattribute__(self, "txn").execute( + return self.txn.execute( sql, *args, **kwargs ) finally: From da1dda3e1d9d3272527d35c23162c4baf7339d74 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Oct 2014 11:18:04 +0000 Subject: [PATCH 022/227] Add transaction level logging and timing information. Add a _simple_delete method --- synapse/storage/__init__.py | 3 +- synapse/storage/_base.py | 74 ++++++++++++++++++++++++++------- synapse/storage/directory.py | 1 + synapse/storage/pdu.py | 13 +++++- synapse/storage/registration.py | 7 +++- synapse/storage/room.py | 2 + synapse/storage/state.py | 1 + synapse/storage/stream.py | 5 ++- synapse/storage/transactions.py | 6 +++ 9 files changed, 91 insertions(+), 21 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 15a72d0cd7..a50e19349a 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -109,6 +109,7 @@ class DataStore(RoomMemberStore, RoomStore, try: yield self.runInteraction( + "persist_event", self._persist_pdu_event_txn, pdu=pdu, event=event, @@ -394,7 +395,7 @@ class DataStore(RoomMemberStore, RoomStore, prev_state_pdu=prev_state_pdu, ) - return self.runInteraction(_snapshot) + return self.runInteraction("snapshot_room", _snapshot) class Snapshot(object): diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index d3e8741889..1192216971 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -29,15 +29,17 @@ import time logger = logging.getLogger(__name__) sql_logger = logging.getLogger("synapse.storage.SQL") +transaction_logger = logging.getLogger("synapse.storage.txn") class LoggingTransaction(object): """An object that almost-transparently proxies for the 'txn' object passed to the constructor. Adds logging to the .execute() method.""" - __slots__ = ["txn"] + __slots__ = ["txn", "name"] - def __init__(self, txn): + def __init__(self, txn, name): object.__setattr__(self, "txn", txn) + object.__setattr__(self, "name", name) def __getattr__(self, name): return getattr(self.txn, name) @@ -47,12 +49,15 @@ class LoggingTransaction(object): def execute(self, sql, *args, **kwargs): # TODO(paul): Maybe use 'info' and 'debug' for values? - sql_logger.debug("[SQL] %s", sql) + sql_logger.debug("[SQL] {%s} %s", self.name, sql) try: if args and args[0]: values = args[0] - sql_logger.debug("[SQL values] " + - ", ".join(("<%s>",) * len(values)), *values) + sql_logger.debug( + "[SQL values] {%s} " + ", ".join(("<%s>",) * len(values)), + self.name, + *values + ) except: # Don't let logging failures stop SQL from working pass @@ -64,10 +69,11 @@ class LoggingTransaction(object): ) finally: end = time.clock() * 1000 - sql_logger.debug("[SQL time] %f", end - start) + sql_logger.debug("[SQL time] {%s} %f", self.name, end - start) class SQLBaseStore(object): + _TXN_ID = 0 def __init__(self, hs): self.hs = hs @@ -75,10 +81,24 @@ class SQLBaseStore(object): self.event_factory = hs.get_event_factory() self._clock = hs.get_clock() - def runInteraction(self, func, *args, **kwargs): + def runInteraction(self, desc, func, *args, **kwargs): """Wraps the .runInteraction() method on the underlying db_pool.""" def inner_func(txn, *args, **kwargs): - return func(LoggingTransaction(txn), *args, **kwargs) + start = time.clock() * 1000 + txn_id = str(SQLBaseStore._TXN_ID) + SQLBaseStore._TXN_ID += 1 + + name = "%s-%s" % (desc, txn_id, ) + + transaction_logger.debug("[TXN START] {%s}", name) + try: + return func(LoggingTransaction(txn, name), *args, **kwargs) + finally: + end = time.clock() * 1000 + transaction_logger.debug( + "[TXN END] {%s} %f", + name, end - start + ) return self._db_pool.runInteraction(inner_func, *args, **kwargs) @@ -114,7 +134,7 @@ class SQLBaseStore(object): else: return cursor.fetchall() - return self.runInteraction(interaction) + return self.runInteraction("_execute", interaction) def _execute_and_decode(self, query, *args): return self._execute(self.cursor_to_dict, query, *args) @@ -131,6 +151,7 @@ class SQLBaseStore(object): or_replace : bool; if True performs an INSERT OR REPLACE """ return self.runInteraction( + "_simple_insert", self._simple_insert_txn, table, values, or_replace=or_replace, or_ignore=or_ignore, ) @@ -168,6 +189,7 @@ class SQLBaseStore(object): statement returns no rows """ return self._simple_selectupdate_one( + "_simple_select_one", table, keyvalues, retcols=retcols, allow_none=allow_none ) @@ -217,7 +239,7 @@ class SQLBaseStore(object): txn.execute(sql, keyvalues.values()) return txn.fetchall() - res = yield self.runInteraction(func) + res = yield self.runInteraction("_simple_select_onecol", func) defer.returnValue([r[0] for r in res]) @@ -240,7 +262,7 @@ class SQLBaseStore(object): txn.execute(sql, keyvalues.values()) return self.cursor_to_dict(txn) - return self.runInteraction(func) + return self.runInteraction("_simple_select_list", func) def _simple_update_one(self, table, keyvalues, updatevalues, retcols=None): @@ -308,7 +330,7 @@ class SQLBaseStore(object): raise StoreError(500, "More than one row matched") return ret - return self.runInteraction(func) + return self.runInteraction("_simple_selectupdate_one", func) def _simple_delete_one(self, table, keyvalues): """Executes a DELETE query on the named table, expecting to delete a @@ -320,7 +342,7 @@ class SQLBaseStore(object): """ sql = "DELETE FROM %s WHERE %s" % ( table, - " AND ".join("%s = ?" % (k) for k in keyvalues) + " AND ".join("%s = ?" % (k, ) for k in keyvalues) ) def func(txn): @@ -329,7 +351,25 @@ class SQLBaseStore(object): raise StoreError(404, "No row found") if txn.rowcount > 1: raise StoreError(500, "more than one row matched") - return self.runInteraction(func) + return self.runInteraction("_simple_delete_one", func) + + def _simple_delete(self, table, keyvalues): + """Executes a DELETE query on the named table. + + Args: + table : string giving the table name + keyvalues : dict of column names and values to select the row with + """ + + return self.runInteraction("_simple_delete", self._simple_delete_txn) + + def _simple_delete_txn(self, txn, table, keyvalues): + sql = "DELETE FROM %s WHERE %s" % ( + table, + " AND ".join("%s = ?" % (k, ) for k in keyvalues) + ) + + return txn.execute(sql, keyvalues.values()) def _simple_max_id(self, table): """Executes a SELECT query on the named table, expecting to return the @@ -347,7 +387,7 @@ class SQLBaseStore(object): return 0 return max_id - return self.runInteraction(func) + return self.runInteraction("_simple_max_id", func) def _parse_event_from_row(self, row_dict): d = copy.deepcopy({k: v for k, v in row_dict.items()}) @@ -371,7 +411,9 @@ class SQLBaseStore(object): ) def _parse_events(self, rows): - return self.runInteraction(self._parse_events_txn, rows) + return self.runInteraction( + "_parse_events", self._parse_events_txn, rows + ) def _parse_events_txn(self, txn, rows): events = [self._parse_event_from_row(r) for r in rows] diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index 52373a28a6..d6a7113b9c 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -95,6 +95,7 @@ class DirectoryStore(SQLBaseStore): def delete_room_alias(self, room_alias): return self.runInteraction( + "delete_room_alias", self._delete_room_alias_txn, room_alias, ) diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index 9bdc831fd8..4a4341907b 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -47,7 +47,7 @@ class PduStore(SQLBaseStore): """ return self.runInteraction( - self._get_pdu_tuple, pdu_id, origin + "get_pdu", self._get_pdu_tuple, pdu_id, origin ) def _get_pdu_tuple(self, txn, pdu_id, origin): @@ -108,6 +108,7 @@ class PduStore(SQLBaseStore): """ return self.runInteraction( + "get_current_state_for_context", self._get_current_state_for_context, context ) @@ -156,6 +157,7 @@ class PduStore(SQLBaseStore): """ return self.runInteraction( + "mark_pdu_as_processed", self._mark_as_processed, pdu_id, pdu_origin ) @@ -165,6 +167,7 @@ class PduStore(SQLBaseStore): def get_all_pdus_from_context(self, context): """Get a list of all PDUs for a given context.""" return self.runInteraction( + "get_all_pdus_from_context", self._get_all_pdus_from_context, context, ) @@ -192,6 +195,7 @@ class PduStore(SQLBaseStore): list: A list of PduTuples """ return self.runInteraction( + "get_backfill", self._get_backfill, context, pdu_list, limit ) @@ -253,6 +257,7 @@ class PduStore(SQLBaseStore): context (str) """ return self.runInteraction( + "get_min_depth_for_context", self._get_min_depth_for_context, context ) @@ -291,6 +296,7 @@ class PduStore(SQLBaseStore): def get_latest_pdus_in_context(self, context): return self.runInteraction( + "get_latest_pdus_in_context", self._get_latest_pdus_in_context, context ) @@ -370,6 +376,7 @@ class PduStore(SQLBaseStore): """ return self.runInteraction( + "is_pdu_new", self._is_pdu_new, pdu_id=pdu_id, origin=origin, @@ -523,6 +530,7 @@ class StatePduStore(SQLBaseStore): def get_unresolved_state_tree(self, new_state_pdu): return self.runInteraction( + "get_unresolved_state_tree", self._get_unresolved_state_tree, new_state_pdu ) @@ -562,6 +570,7 @@ class StatePduStore(SQLBaseStore): def update_current_state(self, pdu_id, origin, context, pdu_type, state_key): return self.runInteraction( + "update_current_state", self._update_current_state, pdu_id, origin, context, pdu_type, state_key ) @@ -601,6 +610,7 @@ class StatePduStore(SQLBaseStore): """ return self.runInteraction( + "get_current_state_pdu", self._get_current_state_pdu, context, pdu_type, state_key ) @@ -660,6 +670,7 @@ class StatePduStore(SQLBaseStore): bool: True if the new_pdu clobbered the current state, False if not """ return self.runInteraction( + "handle_new_state", self._handle_new_state, new_pdu ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 719806f82b..a2ca6f9a69 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -62,8 +62,10 @@ class RegistrationStore(SQLBaseStore): Raises: StoreError if the user_id could not be registered. """ - yield self.runInteraction(self._register, user_id, token, - password_hash) + yield self.runInteraction( + "register", + self._register, user_id, token, password_hash + ) def _register(self, txn, user_id, token, password_hash): now = int(self.clock.time()) @@ -100,6 +102,7 @@ class RegistrationStore(SQLBaseStore): StoreError if no user was found. """ return self.runInteraction( + "get_user_by_token", self._query_for_auth, token ) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 8cd46334cf..7e48ce9cc3 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -150,6 +150,7 @@ class RoomStore(SQLBaseStore): def get_power_level(self, room_id, user_id): return self.runInteraction( + "get_power_level", self._get_power_level, room_id, user_id, ) @@ -183,6 +184,7 @@ class RoomStore(SQLBaseStore): def get_ops_levels(self, room_id): return self.runInteraction( + "get_ops_levels", self._get_ops_levels, room_id, ) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 0aa979c9f0..e08acd6404 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -59,6 +59,7 @@ class StateStore(SQLBaseStore): def store_state_groups(self, event): return self.runInteraction( + "store_state_groups", self._store_state_groups_txn, event ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index d61f909939..8f7f61d29d 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -309,7 +309,10 @@ class StreamStore(SQLBaseStore): defer.returnValue(ret) def get_room_events_max_id(self): - return self.runInteraction(self._get_room_events_max_id_txn) + return self.runInteraction( + "get_room_events_max_id", + self._get_room_events_max_id_txn + ) def _get_room_events_max_id_txn(self, txn): txn.execute( diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 2ba8e30efe..908014d38b 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -42,6 +42,7 @@ class TransactionStore(SQLBaseStore): """ return self.runInteraction( + "get_received_txn_response", self._get_received_txn_response, transaction_id, origin ) @@ -73,6 +74,7 @@ class TransactionStore(SQLBaseStore): """ return self.runInteraction( + "set_received_txn_response", self._set_received_txn_response, transaction_id, origin, code, response_dict ) @@ -106,6 +108,7 @@ class TransactionStore(SQLBaseStore): """ return self.runInteraction( + "prep_send_transaction", self._prep_send_transaction, transaction_id, destination, origin_server_ts, pdu_list ) @@ -161,6 +164,7 @@ class TransactionStore(SQLBaseStore): response_json (str) """ return self.runInteraction( + "delivered_txn", self._delivered_txn, transaction_id, destination, code, response_dict ) @@ -186,6 +190,7 @@ class TransactionStore(SQLBaseStore): list: A list of `ReceivedTransactionsTable.EntryType` """ return self.runInteraction( + "get_transactions_after", self._get_transactions_after, transaction_id, destination ) @@ -216,6 +221,7 @@ class TransactionStore(SQLBaseStore): list: A list of PduTuple """ return self.runInteraction( + "get_pdus_after_transaction", self._get_pdus_after_transaction, transaction_id, destination ) From 2d1dfb3b34583a4de7e1e53f685c2564a7fc731f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Oct 2014 16:42:35 +0000 Subject: [PATCH 023/227] Begin implementing all the PDU storage stuff in Events land --- synapse/api/events/__init__.py | 4 +- synapse/federation/pdu_codec.py | 11 +- synapse/storage/__init__.py | 72 +++++++--- synapse/storage/_base.py | 53 +++++--- synapse/storage/event_federation.py | 143 ++++++++++++++++++++ synapse/storage/schema/event_edges.sql | 51 +++++++ synapse/storage/schema/event_signatures.sql | 65 +++++++++ synapse/storage/schema/im.sql | 1 + synapse/storage/signatures.py | 127 +++++++++++++++++ 9 files changed, 485 insertions(+), 42 deletions(-) create mode 100644 synapse/storage/event_federation.py create mode 100644 synapse/storage/schema/event_edges.sql create mode 100644 synapse/storage/schema/event_signatures.sql diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index a5a55742e0..b855811b98 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -71,7 +71,9 @@ class SynapseEvent(JsonEncodedObject): "outlier", "power_level", "redacted", - "prev_pdus", + "prev_events", + "hashes", + "signatures", ] required_keys = [ diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index 991aae2a56..2cd591410b 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -47,7 +47,10 @@ class PduCodec(object): kwargs["event_id"] = encode_event_id(pdu.pdu_id, pdu.origin) kwargs["room_id"] = pdu.context kwargs["etype"] = pdu.pdu_type - kwargs["prev_pdus"] = pdu.prev_pdus + kwargs["prev_events"] = [ + encode_event_id(i, o) + for i, o in pdu.prev_pdus + ] if hasattr(pdu, "prev_state_id") and hasattr(pdu, "prev_state_origin"): kwargs["prev_state"] = encode_event_id( @@ -78,8 +81,8 @@ class PduCodec(object): d["context"] = event.room_id d["pdu_type"] = event.type - if hasattr(event, "prev_pdus"): - d["prev_pdus"] = event.prev_pdus + if hasattr(event, "prev_events"): + d["prev_pdus"] = [decode_event_id(e) for e in event.prev_events] if hasattr(event, "prev_state"): d["prev_state_id"], d["prev_state_origin"] = ( @@ -92,7 +95,7 @@ class PduCodec(object): kwargs = copy.deepcopy(event.unrecognized_keys) kwargs.update({ k: v for k, v in d.items() - if k not in ["event_id", "room_id", "type"] + if k not in ["event_id", "room_id", "type", "prev_events"] }) if "origin_server_ts" not in kwargs: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index a50e19349a..678de0cf50 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -40,6 +40,7 @@ from .stream import StreamStore from .pdu import StatePduStore, PduStore, PdusTable from .transactions import TransactionStore from .keys import KeyStore +from .event_federation import EventFederationStore from .state import StateStore from .signatures import SignatureStore @@ -69,6 +70,7 @@ SCHEMAS = [ "redactions", "state", "signatures", + "event_edges", ] @@ -83,10 +85,12 @@ class _RollbackButIsFineException(Exception): """ pass + class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, PduStore, StatePduStore, TransactionStore, - DirectoryStore, KeyStore, StateStore, SignatureStore): + DirectoryStore, KeyStore, StateStore, SignatureStore, + EventFederationStore, ): def __init__(self, hs): super(DataStore, self).__init__(hs) @@ -230,6 +234,10 @@ class DataStore(RoomMemberStore, RoomStore, elif event.type == RoomRedactionEvent.TYPE: self._store_redaction(txn, event) + outlier = False + if hasattr(event, "outlier"): + outlier = event.outlier + vals = { "topological_ordering": event.depth, "event_id": event.event_id, @@ -237,20 +245,20 @@ class DataStore(RoomMemberStore, RoomStore, "room_id": event.room_id, "content": json.dumps(event.content), "processed": True, + "outlier": outlier, + "depth": event.depth, } if stream_ordering is not None: vals["stream_ordering"] = stream_ordering - if hasattr(event, "outlier"): - vals["outlier"] = event.outlier - else: - vals["outlier"] = False - unrec = { k: v for k, v in event.get_full_dict().items() - if k not in vals.keys() and k not in ["redacted", "redacted_because"] + if k not in vals.keys() and k not in [ + "redacted", "redacted_because", "signatures", "hashes", + "prev_events", + ] } vals["unrecognized_keys"] = json.dumps(unrec) @@ -264,6 +272,14 @@ class DataStore(RoomMemberStore, RoomStore, ) raise _RollbackButIsFineException("_persist_event") + self._handle_prev_events( + txn, + outlier=outlier, + event_id=event.event_id, + prev_events=event.prev_events, + room_id=event.room_id, + ) + self._store_state_groups_txn(txn, event) is_state = hasattr(event, "state_key") and event.state_key is not None @@ -291,6 +307,28 @@ class DataStore(RoomMemberStore, RoomStore, } ) + signatures = event.signatures.get(event.origin, {}) + + for key_id, signature_base64 in signatures.items(): + signature_bytes = decode_base64(signature_base64) + self._store_event_origin_signature_txn( + txn, event.event_id, key_id, signature_bytes, + ) + + for prev_event_id, prev_hashes in event.prev_events: + for alg, hash_base64 in prev_hashes.items(): + hash_bytes = decode_base64(hash_base64) + self._store_prev_event_hash_txn( + txn, event.event_id, prev_event_id, alg, hash_bytes + ) + + (ref_alg, ref_hash_bytes) = compute_pdu_event_reference_hash(pdu) + self._store_pdu_reference_hash_txn( + txn, pdu.pdu_id, pdu.origin, ref_alg, ref_hash_bytes + ) + + self._update_min_depth_for_room_txn(txn, event.room_id, event.depth) + def _store_redaction(self, txn, event): txn.execute( "INSERT OR IGNORE INTO redactions " @@ -373,7 +411,7 @@ class DataStore(RoomMemberStore, RoomStore, """ def _snapshot(txn): membership_state = self._get_room_member(txn, user_id, room_id) - prev_pdus = self._get_latest_pdus_in_context( + prev_events = self._get_latest_events_in_room( txn, room_id ) @@ -388,7 +426,7 @@ class DataStore(RoomMemberStore, RoomStore, store=self, room_id=room_id, user_id=user_id, - prev_pdus=prev_pdus, + prev_events=prev_events, membership_state=membership_state, state_type=state_type, state_key=state_key, @@ -404,7 +442,7 @@ class Snapshot(object): store (DataStore): The datastore. room_id (RoomId): The room of the snapshot. user_id (UserId): The user this snapshot is for. - prev_pdus (list): The list of PDU ids this snapshot is after. + prev_events (list): The list of event ids this snapshot is after. membership_state (RoomMemberEvent): The current state of the user in the room. state_type (str, optional): State type captured by the snapshot @@ -413,29 +451,29 @@ class Snapshot(object): the previous value of the state type and key in the room. """ - def __init__(self, store, room_id, user_id, prev_pdus, + def __init__(self, store, room_id, user_id, prev_events, membership_state, state_type=None, state_key=None, prev_state_pdu=None): self.store = store self.room_id = room_id self.user_id = user_id - self.prev_pdus = prev_pdus + self.prev_events = prev_events self.membership_state = membership_state self.state_type = state_type self.state_key = state_key self.prev_state_pdu = prev_state_pdu def fill_out_prev_events(self, event): - if hasattr(event, "prev_pdus"): + if hasattr(event, "prev_events"): return - event.prev_pdus = [ + event.prev_events = [ (p_id, origin, hashes) - for p_id, origin, hashes, _ in self.prev_pdus + for p_id, origin, hashes, _ in self.prev_events ] - if self.prev_pdus: - event.depth = max([int(v) for _, _, _, v in self.prev_pdus]) + 1 + if self.prev_events: + event.depth = max([int(v) for _, _, _, v in self.prev_events]) + 1 else: event.depth = 0 diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1192216971..30732caa83 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -193,7 +193,6 @@ class SQLBaseStore(object): table, keyvalues, retcols=retcols, allow_none=allow_none ) - @defer.inlineCallbacks def _simple_select_one_onecol(self, table, keyvalues, retcol, allow_none=False): """Executes a SELECT query on the named table, which is expected to @@ -204,19 +203,41 @@ class SQLBaseStore(object): keyvalues : dict of column names and values to select the row with retcol : string giving the name of the column to return """ - ret = yield self._simple_select_one( + return self.runInteraction( + "_simple_select_one_onecol_txn", + self._simple_select_one_onecol_txn, + table, keyvalues, retcol, allow_none=allow_none, + ) + + def _simple_select_one_onecol_txn(self, txn, table, keyvalues, retcol, + allow_none=False): + ret = self._simple_select_onecol_txn( + txn, table=table, keyvalues=keyvalues, - retcols=[retcol], - allow_none=allow_none + retcols=retcol, ) if ret: - defer.returnValue(ret[retcol]) + return ret[retcol] else: - defer.returnValue(None) + if allow_none: + return None + else: + raise StoreError(404, "No row found") + + def _simple_select_onecol_txn(self, txn, table, keyvalues, retcol): + sql = "SELECT %(retcol)s FROM %(table)s WHERE %(where)s" % { + "retcol": retcol, + "table": table, + "where": " AND ".join("%s = ?" % k for k in keyvalues.keys()), + } + + txn.execute(sql, keyvalues.values()) + + return [r[0] for r in txn.fetchall()] + - @defer.inlineCallbacks def _simple_select_onecol(self, table, keyvalues, retcol): """Executes a SELECT query on the named table, which returns a list comprising of the values of the named column from the selected rows. @@ -229,19 +250,11 @@ class SQLBaseStore(object): Returns: Deferred: Results in a list """ - sql = "SELECT %(retcol)s FROM %(table)s WHERE %(where)s" % { - "retcol": retcol, - "table": table, - "where": " AND ".join("%s = ?" % k for k in keyvalues.keys()), - } - - def func(txn): - txn.execute(sql, keyvalues.values()) - return txn.fetchall() - - res = yield self.runInteraction("_simple_select_onecol", func) - - defer.returnValue([r[0] for r in res]) + return self.runInteraction( + "_simple_select_onecol", + self._simple_select_onecol_txn, + table, keyvalues, retcol + ) def _simple_select_list(self, table, keyvalues, retcols): """Executes a SELECT query on the named table, which may return zero or diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py new file mode 100644 index 0000000000..27ad9aea4d --- /dev/null +++ b/synapse/storage/event_federation.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import SQLBaseStore +from twisted.internet import defer + +import logging + + +logger = logging.getLogger(__name__) + + +class EventFederationStore(SQLBaseStore): + + def _get_latest_events_in_room(self, txn, room_id): + self._simple_select_onecol_txn( + txn, + table="event_forward_extremities", + keyvalues={ + "room_id": room_id, + }, + retcol="event_id", + ) + + results = [] + for pdu_id, origin, depth in txn.fetchall(): + hashes = self._get_pdu_reference_hashes_txn(txn, pdu_id, origin) + sha256_bytes = hashes["sha256"] + prev_hashes = {"sha256": encode_base64(sha256_bytes)} + results.append((pdu_id, origin, prev_hashes, depth)) + + def _get_min_depth_interaction(self, txn, room_id): + min_depth = self._simple_select_one_onecol_txn( + txn, + table="room_depth", + keyvalues={"room_id": room_id,}, + retcol="min_depth", + allow_none=True, + ) + + return int(min_depth) if min_depth is not None else None + + def _update_min_depth_for_room_txn(self, txn, room_id, depth): + min_depth = self._get_min_depth_interaction(txn, room_id) + + do_insert = depth < min_depth if min_depth else True + + if do_insert: + self._simple_insert_txn( + txn, + table="room_depth", + values={ + "room_id": room_id, + "min_depth": depth, + }, + or_replace=True, + ) + + def _handle_prev_events(self, txn, outlier, event_id, prev_events, + room_id): + for e_id in prev_events: + # TODO (erikj): This could be done as a bulk insert + self._simple_insert_txn( + txn, + table="event_edges", + values={ + "event_id": event_id, + "prev_event": e_id, + "room_id": room_id, + } + ) + + # Update the extremities table if this is not an outlier. + if not outlier: + for e_id in prev_events: + # TODO (erikj): This could be done as a bulk insert + self._simple_delete_txn( + txn, + table="event_forward_extremities", + keyvalues={ + "event_id": e_id, + "room_id": room_id, + } + ) + + + + # We only insert as a forward extremity the new pdu if there are no + # other pdus that reference it as a prev pdu + query = ( + "INSERT INTO %(table)s (event_id, room_id) " + "SELECT ?, ? WHERE NOT EXISTS (" + "SELECT 1 FROM %(event_edges)s WHERE " + "prev_event_id = ? " + ")" + ) % { + "table": "event_forward_extremities", + "event_edges": "event_edges", + } + + logger.debug("query: %s", query) + + txn.execute(query, (event_id, room_id, event_id)) + + # Insert all the prev_pdus as a backwards thing, they'll get + # deleted in a second if they're incorrect anyway. + for e_id in prev_events: + # TODO (erikj): This could be done as a bulk insert + self._simple_insert_txn( + txn, + table="event_backward_extremities", + values={ + "event_id": e_id, + "room_id": room_id, + } + ) + + # Also delete from the backwards extremities table all ones that + # reference pdus that we have already seen + query = ( + "DELETE FROM %(event_back)s as b WHERE EXISTS (" + "SELECT 1 FROM %(events)s AS events " + "WHERE " + "b.event_id = events.event_id " + "AND not events.outlier " + ")" + ) % { + "event_back": "event_backward_extremities", + "events": "events", + } + txn.execute(query) \ No newline at end of file diff --git a/synapse/storage/schema/event_edges.sql b/synapse/storage/schema/event_edges.sql new file mode 100644 index 0000000000..6a28314ece --- /dev/null +++ b/synapse/storage/schema/event_edges.sql @@ -0,0 +1,51 @@ + +CREATE TABLE IF NOT EXISTS event_forward_extremities( + event_id TEXT, + room_id TEXT, + CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); +CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); +-- + +CREATE TABLE IF NOT EXISTS event_backward_extremities( + event_id TEXT, + room_id TEXT, + CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); +CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); +-- + +CREATE TABLE IF NOT EXISTS event_edges( + event_id TEXT, + prev_event_id TEXT, + room_id TEXT, + CONSTRAINT uniqueness UNIQUE (event_id, prev_event_id, room_id) +); + +CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); +CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); +-- + + +CREATE TABLE IF NOT EXISTS room_depth( + room_id TEXT, + min_depth INTEGER, + CONSTRAINT uniqueness UNIQUE (room_id) +); + +CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); +-- + +create TABLE IF NOT EXISTS event_destinations( + event_id TEXT, + destination TEXT, + delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered + CONSTRAINT uniqueness UNIQUE (event_id, destination) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); +-- \ No newline at end of file diff --git a/synapse/storage/schema/event_signatures.sql b/synapse/storage/schema/event_signatures.sql new file mode 100644 index 0000000000..5491c7ecec --- /dev/null +++ b/synapse/storage/schema/event_signatures.sql @@ -0,0 +1,65 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS event_content_hashes ( + event_id TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE (event_id, algorithm) +); + +CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes( + event_id +); + + +CREATE TABLE IF NOT EXISTS event_reference_hashes ( + event_id TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE (event_id, algorithm) +); + +CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes ( + event_id +); + + +CREATE TABLE IF NOT EXISTS event_origin_signatures ( + event_id TEXT, + origin TEXT, + key_id TEXT, + signature BLOB, + CONSTRAINT uniqueness UNIQUE (event_id, key_id) +); + +CREATE INDEX IF NOT EXISTS event_origin_signatures_id ON event_origin_signatures ( + event_id +); + + +CREATE TABLE IF NOT EXISTS event_edge_hashes( + event_id TEXT, + prev_event_id TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE ( + event_id, prev_event_id, algorithm + ) +); + +CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes( + event_id +); diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 3aa83f5c8c..8d6f655993 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -23,6 +23,7 @@ CREATE TABLE IF NOT EXISTS events( unrecognized_keys TEXT, processed BOOL NOT NULL, outlier BOOL NOT NULL, + depth INTEGER DEFAULT 0 NOT NULL, CONSTRAINT ev_uniq UNIQUE (event_id) ); diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 82be946d3f..b8f8fd44cb 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -153,3 +153,130 @@ class SignatureStore(SQLBaseStore): "algorithm": algorithm, "hash": buffer(hash_bytes), }) + + ## Events ## + + def _get_event_content_hashes_txn(self, txn, event_id): + """Get all the hashes for a given Event. + Args: + txn (cursor): + event_id (str): Id for the Event. + Returns: + A dict of algorithm -> hash. + """ + query = ( + "SELECT algorithm, hash" + " FROM event_content_hashes" + " WHERE event_id = ?" + ) + txn.execute(query, (event_id, )) + return dict(txn.fetchall()) + + def _store_event_content_hash_txn(self, txn, event_id, algorithm, + hash_bytes): + """Store a hash for a Event + Args: + txn (cursor): + event_id (str): Id for the Event. + algorithm (str): Hashing algorithm. + hash_bytes (bytes): Hash function output bytes. + """ + self._simple_insert_txn(txn, "event_content_hashes", { + "event_id": event_id, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }) + + def _get_event_reference_hashes_txn(self, txn, event_id): + """Get all the hashes for a given PDU. + Args: + txn (cursor): + event_id (str): Id for the Event. + Returns: + A dict of algorithm -> hash. + """ + query = ( + "SELECT algorithm, hash" + " FROM event_reference_hashes" + " WHERE event_id = ?" + ) + txn.execute(query, (event_id, )) + return dict(txn.fetchall()) + + def _store_event_reference_hash_txn(self, txn, event_id, algorithm, + hash_bytes): + """Store a hash for a PDU + Args: + txn (cursor): + event_id (str): Id for the Event. + algorithm (str): Hashing algorithm. + hash_bytes (bytes): Hash function output bytes. + """ + self._simple_insert_txn(txn, "event_reference_hashes", { + "event_id": event_id, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }) + + + def _get_event_origin_signatures_txn(self, txn, event_id): + """Get all the signatures for a given PDU. + Args: + txn (cursor): + event_id (str): Id for the Event. + Returns: + A dict of key_id -> signature_bytes. + """ + query = ( + "SELECT key_id, signature" + " FROM event_origin_signatures" + " WHERE event_id = ? " + ) + txn.execute(query, (event_id, )) + return dict(txn.fetchall()) + + def _store_event_origin_signature_txn(self, txn, event_id, origin, key_id, + signature_bytes): + """Store a signature from the origin server for a PDU. + Args: + txn (cursor): + event_id (str): Id for the Event. + origin (str): origin of the Event. + key_id (str): Id for the signing key. + signature (bytes): The signature. + """ + self._simple_insert_txn(txn, "event_origin_signatures", { + "event_id": event_id, + "origin": origin, + "key_id": key_id, + "signature": buffer(signature_bytes), + }) + + def _get_prev_event_hashes_txn(self, txn, event_id): + """Get all the hashes for previous PDUs of a PDU + Args: + txn (cursor): + event_id (str): Id for the Event. + Returns: + dict of (pdu_id, origin) -> dict of algorithm -> hash_bytes. + """ + query = ( + "SELECT prev_event_id, algorithm, hash" + " FROM event_edge_hashes" + " WHERE event_id = ?" + ) + txn.execute(query, (event_id, )) + results = {} + for prev_event_id, algorithm, hash_bytes in txn.fetchall(): + hashes = results.setdefault(prev_event_id, {}) + hashes[algorithm] = hash_bytes + return results + + def _store_prev_event_hash_txn(self, txn, event_id, prev_event_id, + algorithm, hash_bytes): + self._simple_insert_txn(txn, "event_edge_hashes", { + "event_id": event_id, + "prev_event_id": prev_event_id, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }) \ No newline at end of file From a10c2ec88d98abe035a60ab0027c1914d4ad7d77 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Oct 2014 17:15:32 +0000 Subject: [PATCH 024/227] Don't reference PDU when persisting event --- synapse/storage/__init__.py | 5 +++-- synapse/storage/event_federation.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 678de0cf50..f89e518690 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -322,9 +322,10 @@ class DataStore(RoomMemberStore, RoomStore, txn, event.event_id, prev_event_id, alg, hash_bytes ) + # TODO (ref_alg, ref_hash_bytes) = compute_pdu_event_reference_hash(pdu) - self._store_pdu_reference_hash_txn( - txn, pdu.pdu_id, pdu.origin, ref_alg, ref_hash_bytes + self._store_event_reference_hash_txn( + txn, event.event_id, ref_alg, ref_hash_bytes ) self._update_min_depth_for_room_txn(txn, event.room_id, event.depth) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 27ad9aea4d..7688fc550f 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -14,7 +14,7 @@ # limitations under the License. from ._base import SQLBaseStore -from twisted.internet import defer +from syutil.base64util import encode_base64 import logging @@ -36,7 +36,7 @@ class EventFederationStore(SQLBaseStore): results = [] for pdu_id, origin, depth in txn.fetchall(): - hashes = self._get_pdu_reference_hashes_txn(txn, pdu_id, origin) + hashes = self._get_prev_event_hashes_txn(txn, pdu_id, origin) sha256_bytes = hashes["sha256"] prev_hashes = {"sha256": encode_base64(sha256_bytes)} results.append((pdu_id, origin, prev_hashes, depth)) From e7858b6d7ef37849a3d2d5004743cdd21ec330a8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Oct 2014 16:59:24 +0000 Subject: [PATCH 025/227] Start filling out and using new events tables --- synapse/federation/pdu_codec.py | 12 +++- synapse/handlers/_base.py | 4 ++ synapse/handlers/federation.py | 96 +++++++++++++++----------- synapse/state.py | 11 +-- synapse/storage/__init__.py | 45 +++++++----- synapse/storage/_base.py | 33 +++++++-- synapse/storage/event_federation.py | 49 ++++++++----- synapse/storage/schema/event_edges.sql | 8 +-- 8 files changed, 162 insertions(+), 96 deletions(-) diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index 2cd591410b..dccbccb85b 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -48,8 +48,8 @@ class PduCodec(object): kwargs["room_id"] = pdu.context kwargs["etype"] = pdu.pdu_type kwargs["prev_events"] = [ - encode_event_id(i, o) - for i, o in pdu.prev_pdus + (encode_event_id(i, o), s) + for i, o, s in pdu.prev_pdus ] if hasattr(pdu, "prev_state_id") and hasattr(pdu, "prev_state_origin"): @@ -82,7 +82,13 @@ class PduCodec(object): d["pdu_type"] = event.type if hasattr(event, "prev_events"): - d["prev_pdus"] = [decode_event_id(e) for e in event.prev_events] + def f(e, s): + i, o = decode_event_id(e, self.server_name) + return i, o, s + d["prev_pdus"] = [ + f(e, s) + for e, s in event.prev_events + ] if hasattr(event, "prev_state"): d["prev_state_id"], d["prev_state_origin"] = ( diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index cd6c35f194..787a01efc5 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -16,6 +16,8 @@ from twisted.internet import defer from synapse.api.errors import LimitExceededError +from synapse.util.async import run_on_reactor + class BaseHandler(object): def __init__(self, hs): @@ -45,6 +47,8 @@ class BaseHandler(object): @defer.inlineCallbacks def _on_new_room_event(self, event, snapshot, extra_destinations=[], extra_users=[], suppress_auth=False): + yield run_on_reactor() + snapshot.fill_out_prev_events(event) yield self.state_handler.annotate_state_groups(event) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b575986fc3..5f86ed03fa 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -22,6 +22,7 @@ from synapse.api.constants import Membership from synapse.util.logutils import log_function from synapse.federation.pdu_codec import PduCodec, encode_event_id from synapse.api.errors import SynapseError +from synapse.util.async import run_on_reactor from twisted.internet import defer, reactor @@ -81,6 +82,8 @@ class FederationHandler(BaseHandler): processing. """ + yield run_on_reactor() + pdu = self.pdu_codec.pdu_from_event(event) if not hasattr(pdu, "destinations") or not pdu.destinations: @@ -102,6 +105,8 @@ class FederationHandler(BaseHandler): self.room_queues[event.room_id].append(pdu) return + logger.debug("Processing event: %s", event.event_id) + if state: state = [self.pdu_codec.event_from_pdu(p) for p in state] @@ -216,58 +221,65 @@ class FederationHandler(BaseHandler): assert(event.state_key == joinee) assert(event.room_id == room_id) + event.outlier = False + self.room_queues[room_id] = [] - event.event_id = self.event_factory.create_event_id() - event.content = content - - state = yield self.replication_layer.send_join( - target_host, - self.pdu_codec.pdu_from_event(event) - ) - - state = [self.pdu_codec.event_from_pdu(p) for p in state] - - logger.debug("do_invite_join state: %s", state) - - is_new_state = yield self.state_handler.annotate_state_groups( - event, - state=state - ) - try: - yield self.store.store_room( - room_id=room_id, - room_creator_user_id="", - is_public=False - ) - except: - # FIXME - pass + event.event_id = self.event_factory.create_event_id() + event.content = content - for e in state: - # FIXME: Auth these. - is_new_state = yield self.state_handler.annotate_state_groups( - e, + state = yield self.replication_layer.send_join( + target_host, + self.pdu_codec.pdu_from_event(event) ) + state = [self.pdu_codec.event_from_pdu(p) for p in state] + + logger.debug("do_invite_join state: %s", state) + + is_new_state = yield self.state_handler.annotate_state_groups( + event, + state=state + ) + + logger.debug("do_invite_join event: %s", event) + + try: + yield self.store.store_room( + room_id=room_id, + room_creator_user_id="", + is_public=False + ) + except: + # FIXME + pass + + for e in state: + # FIXME: Auth these. + e.outlier = True + + yield self.state_handler.annotate_state_groups( + e, + ) + + yield self.store.persist_event( + e, + backfilled=False, + is_new_state=False + ) + yield self.store.persist_event( - e, + event, backfilled=False, - is_new_state=False + is_new_state=is_new_state ) + finally: + room_queue = self.room_queues[room_id] + del self.room_queues[room_id] - yield self.store.persist_event( - event, - backfilled=False, - is_new_state=is_new_state - ) - - room_queue = self.room_queues[room_id] - del self.room_queues[room_id] - - for p in room_queue: - yield self.on_receive_pdu(p, backfilled=False) + for p in room_queue: + yield self.on_receive_pdu(p, backfilled=False) defer.returnValue(True) diff --git a/synapse/state.py b/synapse/state.py index cc6a7db96b..993c4f18d3 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -143,7 +143,9 @@ class StateHandler(object): defer.returnValue(False) return - new_state = yield self.resolve_state_groups(event.prev_events) + new_state = yield self.resolve_state_groups( + [e for e, _ in event.prev_events] + ) event.old_state_events = copy.deepcopy(new_state) @@ -157,12 +159,11 @@ class StateHandler(object): @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): - # FIXME: HACK! - pdus = yield self.store.get_latest_pdus_in_context(room_id) + events = yield self.store.get_latest_events_in_room(room_id) event_ids = [ - encode_event_id(pdu_id, origin) - for pdu_id, origin, _ in pdus + e_id + for e_id, _ in events ] res = yield self.resolve_state_groups(event_ids) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f89e518690..d75c366834 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -71,6 +71,7 @@ SCHEMAS = [ "state", "signatures", "event_edges", + "event_signatures", ] @@ -134,7 +135,8 @@ class DataStore(RoomMemberStore, RoomStore, "type", "room_id", "content", - "unrecognized_keys" + "unrecognized_keys", + "depth", ], allow_none=allow_none, ) @@ -263,7 +265,12 @@ class DataStore(RoomMemberStore, RoomStore, vals["unrecognized_keys"] = json.dumps(unrec) try: - self._simple_insert_txn(txn, "events", vals) + self._simple_insert_txn( + txn, + "events", + vals, + or_replace=(not outlier), + ) except: logger.warn( "Failed to persist, probably duplicate: %s", @@ -307,13 +314,14 @@ class DataStore(RoomMemberStore, RoomStore, } ) - signatures = event.signatures.get(event.origin, {}) + if hasattr(event, "signatures"): + signatures = event.signatures.get(event.origin, {}) - for key_id, signature_base64 in signatures.items(): - signature_bytes = decode_base64(signature_base64) - self._store_event_origin_signature_txn( - txn, event.event_id, key_id, signature_bytes, - ) + for key_id, signature_base64 in signatures.items(): + signature_bytes = decode_base64(signature_base64) + self._store_event_origin_signature_txn( + txn, event.event_id, event.origin, key_id, signature_bytes, + ) for prev_event_id, prev_hashes in event.prev_events: for alg, hash_base64 in prev_hashes.items(): @@ -323,10 +331,10 @@ class DataStore(RoomMemberStore, RoomStore, ) # TODO - (ref_alg, ref_hash_bytes) = compute_pdu_event_reference_hash(pdu) - self._store_event_reference_hash_txn( - txn, event.event_id, ref_alg, ref_hash_bytes - ) + # (ref_alg, ref_hash_bytes) = compute_pdu_event_reference_hash(pdu) + # self._store_event_reference_hash_txn( + # txn, event.event_id, ref_alg, ref_hash_bytes + # ) self._update_min_depth_for_room_txn(txn, event.room_id, event.depth) @@ -412,9 +420,7 @@ class DataStore(RoomMemberStore, RoomStore, """ def _snapshot(txn): membership_state = self._get_room_member(txn, user_id, room_id) - prev_events = self._get_latest_events_in_room( - txn, room_id - ) + prev_events = self._get_latest_events_in_room(txn, room_id) if state_type is not None and state_key is not None: prev_state_pdu = self._get_current_state_pdu( @@ -469,12 +475,12 @@ class Snapshot(object): return event.prev_events = [ - (p_id, origin, hashes) - for p_id, origin, hashes, _ in self.prev_events + (event_id, hashes) + for event_id, hashes, _ in self.prev_events ] if self.prev_events: - event.depth = max([int(v) for _, _, _, v in self.prev_events]) + 1 + event.depth = max([int(v) for _, _, v in self.prev_events]) + 1 else: event.depth = 0 @@ -533,9 +539,10 @@ def prepare_database(db_conn): db_conn.commit() else: - sql_script = "BEGIN TRANSACTION;" + sql_script = "BEGIN TRANSACTION;\n" for sql_loc in SCHEMAS: sql_script += read_schema(sql_loc) + sql_script += "\n" sql_script += "COMMIT TRANSACTION;" c.executescript(sql_script) db_conn.commit() diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 30732caa83..464b12f032 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -19,10 +19,12 @@ from twisted.internet import defer from synapse.api.errors import StoreError from synapse.api.events.utils import prune_event from synapse.util.logutils import log_function +from syutil.base64util import encode_base64 import collections import copy import json +import sys import time @@ -67,6 +69,9 @@ class LoggingTransaction(object): return self.txn.execute( sql, *args, **kwargs ) + except: + logger.exception("[SQL FAIL] {%s}", self.name) + raise finally: end = time.clock() * 1000 sql_logger.debug("[SQL time] {%s} %f", self.name, end - start) @@ -85,14 +90,20 @@ class SQLBaseStore(object): """Wraps the .runInteraction() method on the underlying db_pool.""" def inner_func(txn, *args, **kwargs): start = time.clock() * 1000 - txn_id = str(SQLBaseStore._TXN_ID) - SQLBaseStore._TXN_ID += 1 + txn_id = SQLBaseStore._TXN_ID - name = "%s-%s" % (desc, txn_id, ) + # We don't really need these to be unique, so lets stop it from + # growing really large. + self._TXN_ID = (self._TXN_ID + 1) % (sys.maxint - 1) + + name = "%s-%x" % (desc, txn_id, ) transaction_logger.debug("[TXN START] {%s}", name) try: return func(LoggingTransaction(txn, name), *args, **kwargs) + except: + logger.exception("[TXN FAIL] {%s}", name) + raise finally: end = time.clock() * 1000 transaction_logger.debug( @@ -189,7 +200,6 @@ class SQLBaseStore(object): statement returns no rows """ return self._simple_selectupdate_one( - "_simple_select_one", table, keyvalues, retcols=retcols, allow_none=allow_none ) @@ -215,11 +225,11 @@ class SQLBaseStore(object): txn, table=table, keyvalues=keyvalues, - retcols=retcol, + retcol=retcol, ) if ret: - return ret[retcol] + return ret[0] else: if allow_none: return None @@ -434,6 +444,17 @@ class SQLBaseStore(object): sql = "SELECT * FROM events WHERE event_id = ?" for ev in events: + signatures = self._get_event_origin_signatures_txn( + txn, ev.event_id, + ) + + ev.signatures = { + k: encode_base64(v) for k, v in signatures.items() + } + + prev_events = self._get_latest_events_in_room(txn, ev.room_id) + ev.prev_events = [(e_id, s,) for e_id, s, _ in prev_events] + if hasattr(ev, "prev_state"): # Load previous state_content. # TODO: Should we be pulling this out above? diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 7688fc550f..5f94c31818 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -24,6 +24,13 @@ logger = logging.getLogger(__name__) class EventFederationStore(SQLBaseStore): + def get_latest_events_in_room(self, room_id): + return self.runInteraction( + "get_latest_events_in_room", + self._get_latest_events_in_room, + room_id, + ) + def _get_latest_events_in_room(self, txn, room_id): self._simple_select_onecol_txn( txn, @@ -34,12 +41,25 @@ class EventFederationStore(SQLBaseStore): retcol="event_id", ) + sql = ( + "SELECT e.event_id, e.depth FROM events as e " + "INNER JOIN event_forward_extremities as f " + "ON e.event_id = f.event_id " + "WHERE f.room_id = ?" + ) + + txn.execute(sql, (room_id, )) + results = [] - for pdu_id, origin, depth in txn.fetchall(): - hashes = self._get_prev_event_hashes_txn(txn, pdu_id, origin) - sha256_bytes = hashes["sha256"] - prev_hashes = {"sha256": encode_base64(sha256_bytes)} - results.append((pdu_id, origin, prev_hashes, depth)) + for event_id, depth in txn.fetchall(): + hashes = self._get_prev_event_hashes_txn(txn, event_id) + prev_hashes = { + k: encode_base64(v) for k, v in hashes.items() + if k == "sha256" + } + results.append((event_id, prev_hashes, depth)) + + return results def _get_min_depth_interaction(self, txn, room_id): min_depth = self._simple_select_one_onecol_txn( @@ -70,21 +90,21 @@ class EventFederationStore(SQLBaseStore): def _handle_prev_events(self, txn, outlier, event_id, prev_events, room_id): - for e_id in prev_events: + for e_id, _ in prev_events: # TODO (erikj): This could be done as a bulk insert self._simple_insert_txn( txn, table="event_edges", values={ "event_id": event_id, - "prev_event": e_id, + "prev_event_id": e_id, "room_id": room_id, } ) # Update the extremities table if this is not an outlier. if not outlier: - for e_id in prev_events: + for e_id, _ in prev_events: # TODO (erikj): This could be done as a bulk insert self._simple_delete_txn( txn, @@ -116,7 +136,7 @@ class EventFederationStore(SQLBaseStore): # Insert all the prev_pdus as a backwards thing, they'll get # deleted in a second if they're incorrect anyway. - for e_id in prev_events: + for e_id, _ in prev_events: # TODO (erikj): This could be done as a bulk insert self._simple_insert_txn( txn, @@ -130,14 +150,11 @@ class EventFederationStore(SQLBaseStore): # Also delete from the backwards extremities table all ones that # reference pdus that we have already seen query = ( - "DELETE FROM %(event_back)s as b WHERE EXISTS (" - "SELECT 1 FROM %(events)s AS events " + "DELETE FROM event_backward_extremities WHERE EXISTS (" + "SELECT 1 FROM events " "WHERE " - "b.event_id = events.event_id " + "event_backward_extremities.event_id = events.event_id " "AND not events.outlier " ")" - ) % { - "event_back": "event_backward_extremities", - "events": "events", - } + ) txn.execute(query) \ No newline at end of file diff --git a/synapse/storage/schema/event_edges.sql b/synapse/storage/schema/event_edges.sql index 6a28314ece..e5f768c705 100644 --- a/synapse/storage/schema/event_edges.sql +++ b/synapse/storage/schema/event_edges.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); --- + CREATE TABLE IF NOT EXISTS event_backward_extremities( event_id TEXT, @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities( CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); --- + CREATE TABLE IF NOT EXISTS event_edges( event_id TEXT, @@ -28,7 +28,6 @@ CREATE TABLE IF NOT EXISTS event_edges( CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); --- CREATE TABLE IF NOT EXISTS room_depth( @@ -38,7 +37,7 @@ CREATE TABLE IF NOT EXISTS room_depth( ); CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); --- + create TABLE IF NOT EXISTS event_destinations( event_id TEXT, @@ -48,4 +47,3 @@ create TABLE IF NOT EXISTS event_destinations( ); CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); --- \ No newline at end of file From 53216a500d2e0d815581ada87379ecece05b05a5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Oct 2014 17:02:22 +0000 Subject: [PATCH 026/227] Add a run_on_reactor function --- synapse/util/async.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/util/async.py b/synapse/util/async.py index 647ea6142c..bf578f8bfb 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -21,3 +21,10 @@ def sleep(seconds): d = defer.Deferred() reactor.callLater(seconds, d.callback, seconds) return d + + +def run_on_reactor(): + """ This will cause the rest of the function to be invoked upon the next + iteration of the main loop + """ + return sleep(0) \ No newline at end of file From b29517bd013b82302b1a73072da8bfc39564dc1a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Oct 2014 01:21:33 +0000 Subject: [PATCH 027/227] Add a request-id to each log line --- setup.py | 2 +- synapse/app/homeserver.py | 12 ++++- synapse/config/logger.py | 23 ++++++--- synapse/crypto/keyclient.py | 10 ++-- synapse/http/client.py | 26 ++++++----- synapse/http/server.py | 13 +++++- synapse/storage/_base.py | 16 +++++-- synapse/util/async.py | 5 +- synapse/util/logcontext.py | 85 ++++++++++++++++++++++++++++++++++ synapse/util/logutils.py | 1 + tests/util/test_log_context.py | 43 +++++++++++++++++ 11 files changed, 205 insertions(+), 31 deletions(-) create mode 100644 synapse/util/logcontext.py create mode 100644 tests/util/test_log_context.py diff --git a/setup.py b/setup.py index 660efd5b89..74eee31a78 100755 --- a/setup.py +++ b/setup.py @@ -54,6 +54,6 @@ setup( long_description=read("README.rst"), entry_points=""" [console_scripts] - synapse-homeserver=synapse.app.homeserver:run + synapse-homeserver=synapse.app.homeserver:main """ ) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 6394bc27d1..4e74f4d14c 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -33,6 +33,7 @@ from synapse.api.urls import ( ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory +from synapse.util.logcontext import LoggingContext from daemonize import Daemonize import twisted.manhole.telnet @@ -240,7 +241,7 @@ def setup(): daemon = Daemonize( app="synapse-homeserver", pid=config.pid_file, - action=reactor.run, + action=run, auto_close_fds=False, verbose=True, logger=logger, @@ -250,6 +251,13 @@ def setup(): else: reactor.run() +def run(): + with LoggingContext("run") as context: + reactor.run() + +def main(): + with LoggingContext("main") as context: + setup() if __name__ == '__main__': - setup() + main() diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 56cd095433..2a59bf9d15 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -14,7 +14,7 @@ # limitations under the License. from ._base import Config - +from synapse.util.logcontext import LoggingContextFilter from twisted.python.log import PythonLoggingObserver import logging import logging.config @@ -45,7 +45,8 @@ class LoggingConfig(Config): def setup_logging(self): log_format = ( - '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s' + "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s" + " - %(message)s" ) if self.log_config is None: @@ -54,12 +55,20 @@ class LoggingConfig(Config): level = logging.DEBUG # FIXME: we need a logging.WARN for a -q quiet option + logger = logging.getLogger('') + logger.setLevel(level) + formatter = logging.Formatter(log_format) + if self.log_file: + handler = logging.FileHandler(self.log_file) + else: + handler = logging.StreamHandler() + print handler + handler.setFormatter(formatter) - logging.basicConfig( - level=level, - filename=self.log_file, - format=log_format - ) + handler.addFilter(LoggingContextFilter(request="")) + + logger.addHandler(handler) + logger.info("Test") else: logging.config.fileConfig(self.log_config) diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index 7cfec5148e..33fa9ca837 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -18,6 +18,7 @@ from twisted.web.http import HTTPClient from twisted.internet.protocol import Factory from twisted.internet import defer, reactor from synapse.http.endpoint import matrix_endpoint +from synapse.util.logcontext import PreserveLoggingContext import json import logging @@ -36,10 +37,11 @@ def fetch_server_key(server_name, ssl_context_factory): for i in range(5): try: - protocol = yield endpoint.connect(factory) - server_response, server_certificate = yield protocol.remote_key - defer.returnValue((server_response, server_certificate)) - return + with PreserveLoggingContext(): + protocol = yield endpoint.connect(factory) + server_response, server_certificate = yield protocol.remote_key + defer.returnValue((server_response, server_certificate)) + return except Exception as e: logger.exception(e) raise IOError("Cannot get key for %s" % server_name) diff --git a/synapse/http/client.py b/synapse/http/client.py index 46c90dbb76..8bda42364b 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -16,11 +16,14 @@ from twisted.internet import defer, reactor from twisted.internet.error import DNSLookupError -from twisted.web.client import _AgentBase, _URI, readBody, FileBodyProducer, PartialDownloadError +from twisted.web.client import ( + _AgentBase, _URI, readBody, FileBodyProducer, PartialDownloadError +) from twisted.web.http_headers import Headers from synapse.http.endpoint import matrix_endpoint from synapse.util.async import sleep +from synapse.util.logcontext import PreserveLoggingContext from syutil.jsonutil import encode_canonical_json @@ -106,16 +109,17 @@ class BaseHttpClient(object): producer = body_callback(method, url_bytes, headers_dict) try: - response = yield self.agent.request( - destination, - endpoint, - method, - path_bytes, - param_bytes, - query_bytes, - Headers(headers_dict), - producer - ) + with PreserveLoggingContext(): + response = yield self.agent.request( + destination, + endpoint, + method, + path_bytes, + param_bytes, + query_bytes, + Headers(headers_dict), + producer + ) logger.debug("Got response to %s", method) break diff --git a/synapse/http/server.py b/synapse/http/server.py index 8d419c02dd..ed1f1170cb 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -20,6 +20,7 @@ from syutil.jsonutil import ( from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException ) +from synapse.util.logcontext import LoggingContext from twisted.internet import defer, reactor from twisted.web import server, resource @@ -88,9 +89,19 @@ class JsonResource(HttpServer, resource.Resource): def render(self, request): """ This get's called by twisted every time someone sends us a request. """ - self._async_render(request) + self._async_render_with_logging_context(request) return server.NOT_DONE_YET + _request_id = 0 + + @defer.inlineCallbacks + def _async_render_with_logging_context(self, request): + request_id = "%s-%s" % (request.method, JsonResource._request_id) + JsonResource._request_id += 1 + with LoggingContext(request_id) as request_context: + request_context.request = request_id + yield self._async_render(request) + @defer.inlineCallbacks def _async_render(self, request): """ This get's called by twisted every time someone sends us a request. diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 65a86e9056..2faa63904e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.api.errors import StoreError from synapse.api.events.utils import prune_event from synapse.util.logutils import log_function +from synapse.util.logcontext import PreserveLoggingContext, LoggingContext import collections import copy @@ -74,12 +75,19 @@ class SQLBaseStore(object): self.event_factory = hs.get_event_factory() self._clock = hs.get_clock() + @defer.inlineCallbacks def runInteraction(self, func, *args, **kwargs): """Wraps the .runInteraction() method on the underlying db_pool.""" + current_context = LoggingContext.current_context() def inner_func(txn, *args, **kwargs): - return func(LoggingTransaction(txn), *args, **kwargs) - - return self._db_pool.runInteraction(inner_func, *args, **kwargs) + with LoggingContext("runInteraction") as context: + current_context.copy_to(context) + return func(LoggingTransaction(txn), *args, **kwargs) + with PreserveLoggingContext(): + result = yield self._db_pool.runInteraction( + inner_func, *args, **kwargs + ) + defer.returnValue(result) def cursor_to_dict(self, cursor): """Converts a SQL cursor into an list of dicts. @@ -146,7 +154,7 @@ class SQLBaseStore(object): ) logger.debug( - "[SQL] %s Args=%s Func=%s", + "[SQL] %s Args=%s", sql, values.values(), ) diff --git a/synapse/util/async.py b/synapse/util/async.py index 647ea6142c..3d3fbe182c 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -16,8 +16,11 @@ from twisted.internet import defer, reactor +from .logcontext import PreserveLoggingContext +@defer.inlineCallbacks def sleep(seconds): d = defer.Deferred() reactor.callLater(seconds, d.callback, seconds) - return d + with PreserveLoggingContext(): + yield d diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py new file mode 100644 index 0000000000..46a2855a15 --- /dev/null +++ b/synapse/util/logcontext.py @@ -0,0 +1,85 @@ +from functools import wraps + +import threading +import logging + +class LoggingContext(object): + __slots__ = ["parent_context", "name", "__dict__"] + + thread_local = threading.local() + + class Sentinel(object): + __slots__ = [] + def copy_to(self, record): + pass + + sentinel = Sentinel() + + def __init__(self, name=None): + self.parent_context = None + self.name = name + + def __str__(self): + return "%s@%x" % (self.name, id(self)) + + @classmethod + def current_context(cls): + return getattr(cls.thread_local, "current_context", cls.sentinel) + + def __enter__(self): + if self.parent_context is not None: + raise Exception("Attempt to enter logging context multiple times") + self.parent_context = self.current_context() + self.thread_local.current_context = self + return self + + def __exit__(self, type, value, traceback): + if self.thread_local.current_context is not self: + logging.error( + "Current logging context %s is not the expected context %s", + self.thread_local.current_context, + self + ) + self.thread_local.current_context = self.parent_context + self.parent_context = None + + def __getattr__(self, name): + return getattr(self.parent_context, name) + + def copy_to(self, record): + if self.parent_context is not None: + self.parent_context.copy_to(record) + for key, value in self.__dict__.items(): + setattr(record, key, value) + + @classmethod + def wrap_callback(cls, callback): + context = cls.current_context() + @wraps(callback) + def wrapped(*args, **kargs): + cls.thread_local.current_context = context + return callback(*args, **kargs) + return wrapped + + +class LoggingContextFilter(logging.Filter): + def __init__(self, **defaults): + self.defaults = defaults + + def filter(self, record): + context = LoggingContext.current_context() + for key, value in self.defaults.items(): + setattr(record, key, value) + context.copy_to(record) + return True + + +class PreserveLoggingContext(object): + __slots__ = ["current_context"] + def __enter__(self): + self.current_context = LoggingContext.current_context() + + def __exit__(self, type, value, traceback): + LoggingContext.thread_local.current_context = self.current_context + + diff --git a/synapse/util/logutils.py b/synapse/util/logutils.py index fadf0bd510..903a6cf1b3 100644 --- a/synapse/util/logutils.py +++ b/synapse/util/logutils.py @@ -75,6 +75,7 @@ def trace_function(f): linenum = f.func_code.co_firstlineno pathname = f.func_code.co_filename + @wraps(f) def wrapped(*args, **kwargs): name = f.__module__ logger = logging.getLogger(name) diff --git a/tests/util/test_log_context.py b/tests/util/test_log_context.py new file mode 100644 index 0000000000..efa0f28bad --- /dev/null +++ b/tests/util/test_log_context.py @@ -0,0 +1,43 @@ +from twisted.internet import defer +from twisted.internet import reactor +from .. import unittest + +from synapse.util.async import sleep +from synapse.util.logcontext import LoggingContext + +class LoggingContextTestCase(unittest.TestCase): + + def _check_test_key(self, value): + self.assertEquals( + LoggingContext.current_context().test_key, value + ) + + def test_with_context(self): + with LoggingContext() as context_one: + context_one.test_key = "test" + self._check_test_key("test") + + def test_chaining(self): + with LoggingContext() as context_one: + context_one.test_key = "one" + with LoggingContext() as context_two: + self._check_test_key("one") + context_two.test_key = "two" + self._check_test_key("two") + self._check_test_key("one") + + @defer.inlineCallbacks + def test_sleep(self): + @defer.inlineCallbacks + def competing_callback(): + with LoggingContext() as competing_context: + competing_context.test_key = "competing" + yield sleep(0) + self._check_test_key("competing") + + reactor.callLater(0, competing_callback) + + with LoggingContext() as context_one: + context_one.test_key = "one" + yield sleep(0) + self._check_test_key("one") From aa80900a8e8cd9e7305a66cec336a8e150c46651 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 30 Oct 2014 10:11:06 +0000 Subject: [PATCH 028/227] Fix SQL so that accepts we may want to persist events twice. --- synapse/storage/event_federation.py | 8 ++-- synapse/storage/signatures.py | 64 +++++++++++++++++++---------- 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 5f94c31818..88d09d9ba8 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -99,7 +99,8 @@ class EventFederationStore(SQLBaseStore): "event_id": event_id, "prev_event_id": e_id, "room_id": room_id, - } + }, + or_ignore=True, ) # Update the extremities table if this is not an outlier. @@ -120,7 +121,7 @@ class EventFederationStore(SQLBaseStore): # We only insert as a forward extremity the new pdu if there are no # other pdus that reference it as a prev pdu query = ( - "INSERT INTO %(table)s (event_id, room_id) " + "INSERT OR IGNORE INTO %(table)s (event_id, room_id) " "SELECT ?, ? WHERE NOT EXISTS (" "SELECT 1 FROM %(event_edges)s WHERE " "prev_event_id = ? " @@ -144,7 +145,8 @@ class EventFederationStore(SQLBaseStore): values={ "event_id": e_id, "room_id": room_id, - } + }, + or_ignore=True, ) # Also delete from the backwards extremities table all ones that diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index b8f8fd44cb..5e99174fcd 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -181,11 +181,16 @@ class SignatureStore(SQLBaseStore): algorithm (str): Hashing algorithm. hash_bytes (bytes): Hash function output bytes. """ - self._simple_insert_txn(txn, "event_content_hashes", { - "event_id": event_id, - "algorithm": algorithm, - "hash": buffer(hash_bytes), - }) + self._simple_insert_txn( + txn, + "event_content_hashes", + { + "event_id": event_id, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }, + or_ignore=True, + ) def _get_event_reference_hashes_txn(self, txn, event_id): """Get all the hashes for a given PDU. @@ -212,11 +217,16 @@ class SignatureStore(SQLBaseStore): algorithm (str): Hashing algorithm. hash_bytes (bytes): Hash function output bytes. """ - self._simple_insert_txn(txn, "event_reference_hashes", { - "event_id": event_id, - "algorithm": algorithm, - "hash": buffer(hash_bytes), - }) + self._simple_insert_txn( + txn, + "event_reference_hashes", + { + "event_id": event_id, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }, + or_ignore=True, + ) def _get_event_origin_signatures_txn(self, txn, event_id): @@ -245,12 +255,17 @@ class SignatureStore(SQLBaseStore): key_id (str): Id for the signing key. signature (bytes): The signature. """ - self._simple_insert_txn(txn, "event_origin_signatures", { - "event_id": event_id, - "origin": origin, - "key_id": key_id, - "signature": buffer(signature_bytes), - }) + self._simple_insert_txn( + txn, + "event_origin_signatures", + { + "event_id": event_id, + "origin": origin, + "key_id": key_id, + "signature": buffer(signature_bytes), + }, + or_ignore=True, + ) def _get_prev_event_hashes_txn(self, txn, event_id): """Get all the hashes for previous PDUs of a PDU @@ -274,9 +289,14 @@ class SignatureStore(SQLBaseStore): def _store_prev_event_hash_txn(self, txn, event_id, prev_event_id, algorithm, hash_bytes): - self._simple_insert_txn(txn, "event_edge_hashes", { - "event_id": event_id, - "prev_event_id": prev_event_id, - "algorithm": algorithm, - "hash": buffer(hash_bytes), - }) \ No newline at end of file + self._simple_insert_txn( + txn, + "event_edge_hashes", + { + "event_id": event_id, + "prev_event_id": prev_event_id, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }, + or_ignore=True, + ) \ No newline at end of file From fa955cc2a4666c94c2c23dd2d8f8c89b8dd37e9d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Oct 2014 10:13:46 +0000 Subject: [PATCH 029/227] Pep8 and a few doc strings --- synapse/config/logger.py | 6 ++--- synapse/util/logcontext.py | 51 +++++++++++++++++++++++++++----------- 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 2a59bf9d15..8566296433 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -19,6 +19,7 @@ from twisted.python.log import PythonLoggingObserver import logging import logging.config + class LoggingConfig(Config): def __init__(self, args): super(LoggingConfig, self).__init__(args) @@ -52,9 +53,9 @@ class LoggingConfig(Config): level = logging.INFO if self.verbosity: - level = logging.DEBUG + level = logging.DEBUG - # FIXME: we need a logging.WARN for a -q quiet option + # FIXME: we need a logging.WARN for a -q quiet option logger = logging.getLogger('') logger.setLevel(level) formatter = logging.Formatter(log_format) @@ -62,7 +63,6 @@ class LoggingConfig(Config): handler = logging.FileHandler(self.log_file) else: handler = logging.StreamHandler() - print handler handler.setFormatter(formatter) handler.addFilter(LoggingContextFilter(request="")) diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index 46a2855a15..13176b05ce 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -1,15 +1,23 @@ -from functools import wraps - import threading import logging + class LoggingContext(object): + """Additional context for log formatting. Contexts are scoped within a + "with" block. Contexts inherit the state of their parent contexts. + Args: + name (str): Name for the context for debugging. + """ + __slots__ = ["parent_context", "name", "__dict__"] thread_local = threading.local() class Sentinel(object): + """Sentinel to represent the root context""" + __slots__ = [] + def copy_to(self, record): pass @@ -20,13 +28,15 @@ class LoggingContext(object): self.name = name def __str__(self): - return "%s@%x" % (self.name, id(self)) + return "%s@%x" % (self.name, id(self)) @classmethod def current_context(cls): + """Get the current logging context from thread local storage""" return getattr(cls.thread_local, "current_context", cls.sentinel) def __enter__(self): + """Enters this logging context into thread local storage""" if self.parent_context is not None: raise Exception("Attempt to enter logging context multiple times") self.parent_context = self.current_context() @@ -34,6 +44,11 @@ class LoggingContext(object): return self def __exit__(self, type, value, traceback): + """Restore the logging context in thread local storage to the state it + was before this context was entered. + Returns: + None to avoid suppressing any exeptions that were thrown. + """ if self.thread_local.current_context is not self: logging.error( "Current logging context %s is not the expected context %s", @@ -44,29 +59,32 @@ class LoggingContext(object): self.parent_context = None def __getattr__(self, name): + """Delegate member lookup to parent context""" return getattr(self.parent_context, name) def copy_to(self, record): + """Copy fields from this context and its parents to the record""" if self.parent_context is not None: self.parent_context.copy_to(record) for key, value in self.__dict__.items(): setattr(record, key, value) - @classmethod - def wrap_callback(cls, callback): - context = cls.current_context() - @wraps(callback) - def wrapped(*args, **kargs): - cls.thread_local.current_context = context - return callback(*args, **kargs) - return wrapped - class LoggingContextFilter(logging.Filter): + """Logging filter that adds values from the current logging context to each + record. + Args: + **defaults: Default values to avoid formatters complaining about + missing fields + """ def __init__(self, **defaults): self.defaults = defaults def filter(self, record): + """Add each fields from the logging contexts to the record. + Returns: + True to include the record in the log output. + """ context = LoggingContext.current_context() for key, value in self.defaults.items(): setattr(record, key, value) @@ -75,11 +93,16 @@ class LoggingContextFilter(logging.Filter): class PreserveLoggingContext(object): + """Captures the current logging context and restores it when the scope is + exited. Used to restore the context after a function using + @defer.inlineCallbacks is resumed by a callback from the reactor.""" + __slots__ = ["current_context"] + def __enter__(self): + """Captures the current logging context""" self.current_context = LoggingContext.current_context() def __exit__(self, type, value, traceback): + """Restores the current logging context""" LoggingContext.thread_local.current_context = self.current_context - - From 7a756e5d9d4c5a636d13b0959b72bc4d1518c5be Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Oct 2014 11:15:39 +0000 Subject: [PATCH 030/227] Remove unused 'context' variables to appease pyflakes --- synapse/app/homeserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 4e74f4d14c..17926be88c 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -252,11 +252,11 @@ def setup(): reactor.run() def run(): - with LoggingContext("run") as context: + with LoggingContext("run"): reactor.run() def main(): - with LoggingContext("main") as context: + with LoggingContext("main"): setup() if __name__ == '__main__': From da511334d26a3e2e6e3b8e3d57259896bcd4bb4c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 30 Oct 2014 11:53:35 +0000 Subject: [PATCH 031/227] Make federation return the old current state, so that we can use it to do auth --- synapse/handlers/federation.py | 28 +++++++++++++++++++++++----- synapse/state.py | 14 +++++++++----- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 5f86ed03fa..da99a4b449 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -112,7 +112,7 @@ class FederationHandler(BaseHandler): is_new_state = yield self.state_handler.annotate_state_groups( event, - state=state + old_state=state ) logger.debug("Event: %s", event) @@ -240,7 +240,7 @@ class FederationHandler(BaseHandler): is_new_state = yield self.state_handler.annotate_state_groups( event, - state=state + old_state=state ) logger.debug("do_invite_join event: %s", event) @@ -279,7 +279,10 @@ class FederationHandler(BaseHandler): del self.room_queues[room_id] for p in room_queue: - yield self.on_receive_pdu(p, backfilled=False) + try: + yield self.on_receive_pdu(p, backfilled=False) + except: + pass defer.returnValue(True) @@ -355,15 +358,30 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def get_state_for_pdu(self, pdu_id, pdu_origin): + event_id = encode_event_id(pdu_id, pdu_origin) + state_groups = yield self.store.get_state_groups( - [encode_event_id(pdu_id, pdu_origin)] + [event_id] ) if state_groups: + results = { + (e.type, e.state_key): e for e in state_groups[0].state + } + + event = yield self.store.get_event(event_id) + if hasattr(event, "state_key"): + # Get previous state + if hasattr(event, "prev_state") and event.prev_state: + prev_event = yield self.store.get_event(event.prev_state) + results[(event.type, event.state_key)] = prev_event + else: + del results[(event.type, event.state_key)] + defer.returnValue( [ self.pdu_codec.pdu_from_event(s) - for s in state_groups[0].state + for s in results.values() ] ) else: diff --git a/synapse/state.py b/synapse/state.py index 993c4f18d3..a59688e3b4 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -128,11 +128,15 @@ class StateHandler(object): @defer.inlineCallbacks @log_function - def annotate_state_groups(self, event, state=None): - if state: + def annotate_state_groups(self, event, old_state=None): + if old_state: event.state_group = None - event.old_state_events = None - event.state_events = {(s.type, s.state_key): s for s in state} + event.old_state_events = old_state + event.state_events = {(s.type, s.state_key): s for s in old_state} + + if hasattr(event, "state_key"): + event.state_events[(event.type, event.state_key)] = event + defer.returnValue(False) return @@ -163,7 +167,7 @@ class StateHandler(object): event_ids = [ e_id - for e_id, _ in events + for e_id, _, _ in events ] res = yield self.resolve_state_groups(event_ids) From 12ce441e67a40bb73ac5aca0283c2fe4afac4021 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 30 Oct 2014 17:00:11 +0000 Subject: [PATCH 032/227] Convert event ids to be of the form :example.com --- synapse/api/events/factory.py | 6 +++++- synapse/federation/pdu_codec.py | 35 +++++++++++++++------------------ synapse/handlers/federation.py | 7 +++++-- synapse/server.py | 7 ++++++- synapse/state.py | 17 +++++++++++----- synapse/types.py | 10 ++++++++++ 6 files changed, 54 insertions(+), 28 deletions(-) diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index 06f3bf232b..750096c618 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -21,6 +21,8 @@ from synapse.api.events.room import ( RoomRedactionEvent, ) +from synapse.types import EventID + from synapse.util.stringutils import random_string @@ -59,7 +61,9 @@ class EventFactory(object): local_part = str(int(self.clock.time())) + i + random_string(5) - return "%s@%s" % (local_part, self.hs.hostname) + e_id = EventID.create_local(local_part, self.hs) + + return e_id.to_string() def create_event(self, etype=None, **kwargs): kwargs["type"] = etype diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index dccbccb85b..6d31286290 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -17,22 +17,11 @@ from .units import Pdu from synapse.crypto.event_signing import ( add_event_pdu_content_hash, sign_event_pdu ) +from synapse.types import EventID import copy -def decode_event_id(event_id, server_name): - parts = event_id.split("@") - if len(parts) < 2: - return (event_id, server_name) - else: - return (parts[0], "".join(parts[1:])) - - -def encode_event_id(pdu_id, origin): - return "%s@%s" % (pdu_id, origin) - - class PduCodec(object): def __init__(self, hs): @@ -40,20 +29,28 @@ class PduCodec(object): self.server_name = hs.hostname self.event_factory = hs.get_event_factory() self.clock = hs.get_clock() + self.hs = hs + + def encode_event_id(self, local, domain): + return EventID.create(local, domain, self.hs).to_string() + + def decode_event_id(self, event_id): + e_id = self.hs.parse_eventid(event_id) + return e_id.localpart, e_id.domain def event_from_pdu(self, pdu): kwargs = {} - kwargs["event_id"] = encode_event_id(pdu.pdu_id, pdu.origin) + kwargs["event_id"] = self.encode_event_id(pdu.pdu_id, pdu.origin) kwargs["room_id"] = pdu.context kwargs["etype"] = pdu.pdu_type kwargs["prev_events"] = [ - (encode_event_id(i, o), s) + (self.encode_event_id(i, o), s) for i, o, s in pdu.prev_pdus ] if hasattr(pdu, "prev_state_id") and hasattr(pdu, "prev_state_origin"): - kwargs["prev_state"] = encode_event_id( + kwargs["prev_state"] = self.encode_event_id( pdu.prev_state_id, pdu.prev_state_origin ) @@ -75,15 +72,15 @@ class PduCodec(object): def pdu_from_event(self, event): d = event.get_full_dict() - d["pdu_id"], d["origin"] = decode_event_id( - event.event_id, self.server_name + d["pdu_id"], d["origin"] = self.decode_event_id( + event.event_id ) d["context"] = event.room_id d["pdu_type"] = event.type if hasattr(event, "prev_events"): def f(e, s): - i, o = decode_event_id(e, self.server_name) + i, o = self.decode_event_id(e) return i, o, s d["prev_pdus"] = [ f(e, s) @@ -92,7 +89,7 @@ class PduCodec(object): if hasattr(event, "prev_state"): d["prev_state_id"], d["prev_state_origin"] = ( - decode_event_id(event.prev_state, self.server_name) + self.decode_event_id(event.prev_state) ) if hasattr(event, "state_key"): diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index da99a4b449..1daeee833b 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -20,9 +20,10 @@ from ._base import BaseHandler from synapse.api.events.room import InviteJoinEvent, RoomMemberEvent from synapse.api.constants import Membership from synapse.util.logutils import log_function -from synapse.federation.pdu_codec import PduCodec, encode_event_id +from synapse.federation.pdu_codec import PduCodec from synapse.api.errors import SynapseError from synapse.util.async import run_on_reactor +from synapse.types import EventID from twisted.internet import defer, reactor @@ -358,7 +359,9 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def get_state_for_pdu(self, pdu_id, pdu_origin): - event_id = encode_event_id(pdu_id, pdu_origin) + yield run_on_reactor() + + event_id = EventID.create(pdu_id, pdu_origin, self.hs).to_string() state_groups = yield self.store.get_state_groups( [event_id] diff --git a/synapse/server.py b/synapse/server.py index a4d2d4aba5..d770b20b19 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -28,7 +28,7 @@ from synapse.handlers import Handlers from synapse.rest import RestServletFactory from synapse.state import StateHandler from synapse.storage import DataStore -from synapse.types import UserID, RoomAlias, RoomID +from synapse.types import UserID, RoomAlias, RoomID, EventID from synapse.util import Clock from synapse.util.distributor import Distributor from synapse.util.lockutils import LockManager @@ -143,6 +143,11 @@ class BaseHomeServer(object): object.""" return RoomID.from_string(s, hs=self) + def parse_eventid(self, s): + """Parse the string given by 's' as a Event ID and return a EventID + object.""" + return EventID.from_string(s, hs=self) + def serialize_event(self, e): return serialize_event(self, e) diff --git a/synapse/state.py b/synapse/state.py index a59688e3b4..414701b272 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -16,8 +16,10 @@ from twisted.internet import defer -from synapse.federation.pdu_codec import encode_event_id, decode_event_id from synapse.util.logutils import log_function +from synapse.util.async import run_on_reactor + +from synapse.types import EventID from collections import namedtuple @@ -43,6 +45,7 @@ class StateHandler(object): self.store = hs.get_datastore() self._replication = hs.get_replication_layer() self.server_name = hs.hostname + self.hs = hs @defer.inlineCallbacks @log_function @@ -77,15 +80,17 @@ class StateHandler(object): current_state = snapshot.prev_state_pdu if current_state: - event.prev_state = encode_event_id( - current_state.pdu_id, current_state.origin - ) + event.prev_state = EventID.create( + current_state.pdu_id, current_state.origin, self.hs + ).to_string() # TODO check current_state to see if the min power level is less # than the power level of the user # power_level = self._get_power_level_for_event(event) - pdu_id, origin = decode_event_id(event.event_id, self.server_name) + e_id = self.hs.parse_eventid(event.event_id) + pdu_id = e_id.localpart + origin = e_id.domain yield self.store.update_current_state( pdu_id=pdu_id, @@ -129,6 +134,8 @@ class StateHandler(object): @defer.inlineCallbacks @log_function def annotate_state_groups(self, event, old_state=None): + yield run_on_reactor() + if old_state: event.state_group = None event.old_state_events = old_state diff --git a/synapse/types.py b/synapse/types.py index c51bc8e4f2..649ff2f7d7 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -78,6 +78,11 @@ class DomainSpecificString( """Create a structure on the local domain""" return cls(localpart=localpart, domain=hs.hostname, is_mine=True) + @classmethod + def create(cls, localpart, domain, hs): + is_mine = domain == hs.hostname + return cls(localpart=localpart, domain=domain, is_mine=is_mine) + class UserID(DomainSpecificString): """Structure representing a user ID.""" @@ -94,6 +99,11 @@ class RoomID(DomainSpecificString): SIGIL = "!" +class EventID(DomainSpecificString): + """Structure representing an event id. """ + SIGIL = "$" + + class StreamToken( namedtuple( "Token", From 4a1597f295cfd5ae3879d78bcaa420e0e5ace3b4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 09:48:59 +0000 Subject: [PATCH 033/227] Fix bug in redaction auth. This caused a 500 when sending a redaction due to a typo in a method invocation. --- synapse/api/auth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 93a3533304..c684265101 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -415,8 +415,8 @@ class Auth(object): else: user_level = 0 - _, _, redact_level = self.store._get_ops_level_from_event_state( - event.room_id + _, _, redact_level = self._get_ops_level_from_event_state( + event ) if not redact_level: From f2de2d644af80557baebf43f64f3968b8ab46d0b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 09:59:02 +0000 Subject: [PATCH 034/227] Move the impl of backfill to use events. --- synapse/federation/replication.py | 6 +- synapse/handlers/federation.py | 27 ++++++++- synapse/storage/event_federation.py | 86 ++++++++++++++++++++++++++++- 3 files changed, 114 insertions(+), 5 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 000a3081c2..1628a56294 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -181,7 +181,7 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def backfill(self, dest, context, limit): + def backfill(self, dest, context, limit, extremities): """Requests some more historic PDUs for the given context from the given destination server. @@ -189,12 +189,12 @@ class ReplicationLayer(object): dest (str): The remote home server to ask. context (str): The context to backfill. limit (int): The maximum number of PDUs to return. + extremities (list): List of PDU id and origins of the first pdus + we have seen from the context Returns: Deferred: Results in the received PDUs. """ - extremities = yield self.store.get_oldest_pdus_in_context(context) - logger.debug("backfill extrem=%s", extremities) # If there are no extremeties then we've (probably) reached the start. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 1daeee833b..9f457ce292 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -181,7 +181,17 @@ class FederationHandler(BaseHandler): @log_function @defer.inlineCallbacks def backfill(self, dest, room_id, limit): - pdus = yield self.replication_layer.backfill(dest, room_id, limit) + extremities = yield self.store.get_oldest_events_in_room(room_id) + + pdus = yield self.replication_layer.backfill( + dest, + room_id, + limit, + extremities=[ + self.pdu_codec.decode_event_id(e) + for e in extremities + ] + ) events = [] @@ -390,6 +400,21 @@ class FederationHandler(BaseHandler): else: defer.returnValue([]) + @defer.inlineCallbacks + @log_function + def on_backfill_request(self, context, pdu_list, limit): + + events = yield self.store.get_backfill_events( + context, + [self.pdu_codec.encode_event_id(i, o) for i, o in pdu_list], + limit + ) + + defer.returnValue([ + self.pdu_codec.pdu_from_event(e) + for e in events + ]) + @log_function def _on_user_joined(self, user, room_id): waiters = self.waiting_for_join_list.get((user.to_string(), room_id), []) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 88d09d9ba8..438b42c1da 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -24,6 +24,23 @@ logger = logging.getLogger(__name__) class EventFederationStore(SQLBaseStore): + def get_oldest_events_in_room(self, room_id): + return self.runInteraction( + "get_oldest_events_in_room", + self._get_oldest_events_in_room_txn, + room_id, + ) + + def _get_oldest_events_in_room_txn(self, txn, room_id): + return self._simple_select_onecol_txn( + txn, + table="event_backward_extremities", + keyvalues={ + "room_id": room_id, + }, + retcol="event_id", + ) + def get_latest_events_in_room(self, room_id): return self.runInteraction( "get_latest_events_in_room", @@ -159,4 +176,71 @@ class EventFederationStore(SQLBaseStore): "AND not events.outlier " ")" ) - txn.execute(query) \ No newline at end of file + txn.execute(query) + + + def get_backfill_events(self, room_id, event_list, limit): + """Get a list of Events for a given topic that occured before (and + including) the pdus in pdu_list. Return a list of max size `limit`. + + Args: + txn + room_id (str) + event_list (list) + limit (int) + + Return: + list: A list of PduTuples + """ + return self.runInteraction( + "get_backfill_events", + self._get_backfill_events, room_id, event_list, limit + ) + + def _get_backfill_events(self, txn, room_id, event_list, limit): + logger.debug( + "_get_backfill_events: %s, %s, %s", + room_id, repr(event_list), limit + ) + + # We seed the pdu_results with the things from the pdu_list. + event_results = event_list + + front = event_list + + query = ( + "SELECT prev_event_id FROM event_edges " + "WHERE room_id = ? AND event_id = ? " + "LIMIT ?" + ) + + # We iterate through all event_ids in `front` to select their previous + # events. These are dumped in `new_front`. + # We continue until we reach the limit *or* new_front is empty (i.e., + # we've run out of things to select + while front and len(event_results) < limit: + + new_front = [] + for event_id in front: + logger.debug( + "_backfill_interaction: id=%s", + event_id + ) + + txn.execute( + query, + (room_id, event_id, limit - len(event_results)) + ) + + for row in txn.fetchall(): + logger.debug( + "_backfill_interaction: got id=%s", + *row + ) + new_front.append(row) + + front = new_front + event_results += new_front + + # We also want to update the `prev_pdus` attributes before returning. + return self._get_pdu_tuples(txn, event_results) From 841df4da71b46dc1360f6a71e5ccd9e267211e38 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 09:59:59 +0000 Subject: [PATCH 035/227] Don't store any PDUs --- synapse/federation/replication.py | 45 ++++++++++++++++--------------- synapse/storage/transactions.py | 18 ++++++------- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 1628a56294..89dbf3e2e9 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -57,7 +57,7 @@ class ReplicationLayer(object): self.transport_layer.register_request_handler(self) self.store = hs.get_datastore() - self.pdu_actions = PduActions(self.store) + # self.pdu_actions = PduActions(self.store) self.transaction_actions = TransactionActions(self.store) self._transaction_queue = _TransactionQueue( @@ -278,16 +278,16 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function def on_context_pdus_request(self, context): - pdus = yield self.pdu_actions.get_all_pdus_from_context( - context + raise NotImplementedError( + "on_context_pdus_request is a security violation" ) - defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) @defer.inlineCallbacks @log_function def on_backfill_request(self, context, versions, limit): - - pdus = yield self.pdu_actions.backfill(context, versions, limit) + pdus = yield self.handler.on_backfill_request( + context, versions, limit + ) defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) @@ -383,20 +383,22 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function def on_pull_request(self, origin, versions): - transaction_id = max([int(v) for v in versions]) + raise NotImplementedError("Pull transacions not implemented") - response = yield self.pdu_actions.after_transaction( - transaction_id, - origin, - self.server_name - ) - - if not response: - response = [] - - defer.returnValue( - (200, self._transaction_from_pdus(response).get_dict()) - ) + # transaction_id = max([int(v) for v in versions]) + # + # response = yield self.pdu_actions.after_transaction( + # transaction_id, + # origin, + # self.server_name + # ) + # + # if not response: + # response = [] + # + # defer.returnValue( + # (200, self._transaction_from_pdus(response).get_dict()) + # ) @defer.inlineCallbacks def on_query_request(self, query_type, args): @@ -498,8 +500,7 @@ class ReplicationLayer(object): state = None # Get missing pdus if necessary. - is_new = yield self.pdu_actions.is_new(pdu) - if is_new and not pdu.outlier: + if not pdu.outlier: # We only backfill backwards to the min depth. min_depth = yield self.store.get_min_depth_for_context(pdu.context) @@ -539,7 +540,7 @@ class ReplicationLayer(object): else: ret = None - yield self.pdu_actions.mark_as_processed(pdu) + # yield self.pdu_actions.mark_as_processed(pdu) defer.returnValue(ret) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 908014d38b..6624348fd0 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -142,15 +142,15 @@ class TransactionStore(SQLBaseStore): # Update the tx id -> pdu id mapping - values = [ - (transaction_id, destination, pdu[0], pdu[1]) - for pdu in pdu_list - ] - - logger.debug("Inserting: %s", repr(values)) - - query = TransactionsToPduTable.insert_statement() - txn.executemany(query, values) + # values = [ + # (transaction_id, destination, pdu[0], pdu[1]) + # for pdu in pdu_list + # ] + # + # logger.debug("Inserting: %s", repr(values)) + # + # query = TransactionsToPduTable.insert_statement() + # txn.executemany(query, values) return prev_txns From d84f5b30b8dbccf4bba22c09a08d184fa1911b45 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 10:47:04 +0000 Subject: [PATCH 036/227] old_state_events should be a dict not list --- synapse/state.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 414701b272..c514b118ae 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -138,8 +138,10 @@ class StateHandler(object): if old_state: event.state_group = None - event.old_state_events = old_state - event.state_events = {(s.type, s.state_key): s for s in old_state} + event.old_state_events = { + (s.type, s.state_key): s for s in old_state + } + event.state_events = event.old_state_events if hasattr(event, "state_key"): event.state_events[(event.type, event.state_key)] = event From 21fe249d62deafceca05cc114d5d6bec3e815b8c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 10:47:34 +0000 Subject: [PATCH 037/227] Actually don't store any PDUs --- synapse/federation/replication.py | 27 +++++++++++++-------------- synapse/handlers/federation.py | 22 ++++++++++++++++++++++ synapse/storage/event_federation.py | 7 +++++++ 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 89dbf3e2e9..a0bd2e0572 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -106,7 +106,6 @@ class ReplicationLayer(object): self.query_handlers[query_type] = handler - @defer.inlineCallbacks @log_function def send_pdu(self, pdu): """Informs the replication layer about a new PDU generated within the @@ -135,7 +134,7 @@ class ReplicationLayer(object): logger.debug("[%s] Persisting PDU", pdu.pdu_id) # Save *before* trying to send - yield self.store.persist_event(pdu=pdu) + # yield self.store.persist_event(pdu=pdu) logger.debug("[%s] Persisted PDU", pdu.pdu_id) logger.debug("[%s] transaction_layer.enqueue_pdu... ", pdu.pdu_id) @@ -359,12 +358,13 @@ class ReplicationLayer(object): pdu_id, pdu_origin ) else: - results = yield self.store.get_current_state_for_context( - context - ) - pdus = [Pdu.from_pdu_tuple(p) for p in results] - - logger.debug("Context returning %d results", len(pdus)) + raise NotImplementedError("Specify an event") + # results = yield self.store.get_current_state_for_context( + # context + # ) + # pdus = [Pdu.from_pdu_tuple(p) for p in results] + # + # logger.debug("Context returning %d results", len(pdus)) defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) @@ -456,7 +456,6 @@ class ReplicationLayer(object): defer.returnValue(pdus) - @defer.inlineCallbacks @log_function def _get_persisted_pdu(self, pdu_id, pdu_origin): """ Get a PDU from the database with given origin and id. @@ -464,9 +463,7 @@ class ReplicationLayer(object): Returns: Deferred: Results in a `Pdu`. """ - pdu_tuple = yield self.store.get_pdu(pdu_id, pdu_origin) - - defer.returnValue(Pdu.from_pdu_tuple(pdu_tuple)) + return self.handler.get_persisted_pdu(pdu_id, pdu_origin) def _transaction_from_pdus(self, pdu_list): """Returns a new Transaction containing the given PDUs suitable for @@ -502,7 +499,9 @@ class ReplicationLayer(object): # Get missing pdus if necessary. if not pdu.outlier: # We only backfill backwards to the min depth. - min_depth = yield self.store.get_min_depth_for_context(pdu.context) + min_depth = yield self.handler.get_min_depth_for_context( + pdu.context + ) if min_depth and pdu.depth > min_depth: for pdu_id, origin, hashes in pdu.prev_pdus: @@ -529,7 +528,7 @@ class ReplicationLayer(object): ) # Persist the Pdu, but don't mark it as processed yet. - yield self.store.persist_event(pdu=pdu) + # yield self.store.persist_event(pdu=pdu) if not backfilled: ret = yield self.handler.on_receive_pdu( diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 9f457ce292..18cb1d4e97 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -415,6 +415,28 @@ class FederationHandler(BaseHandler): for e in events ]) + @defer.inlineCallbacks + @log_function + def get_persisted_pdu(self, pdu_id, origin): + """ Get a PDU from the database with given origin and id. + + Returns: + Deferred: Results in a `Pdu`. + """ + event = yield self.store.get_event( + self.pdu_codec.encode_event_id(pdu_id, origin), + allow_none=True, + ) + + if event: + defer.returnValue(self.pdu_codec.pdu_from_event(event)) + else: + defer.returnValue(None) + + @log_function + def get_min_depth_for_context(self, context): + return self.store.get_min_depth(context) + @log_function def _on_user_joined(self, user, room_id): waiters = self.waiting_for_join_list.get((user.to_string(), room_id), []) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 438b42c1da..8357071db6 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -78,6 +78,13 @@ class EventFederationStore(SQLBaseStore): return results + def get_min_depth(self, room_id): + return self.runInteraction( + "get_min_depth", + self._get_min_depth_interaction, + room_id, + ) + def _get_min_depth_interaction(self, txn, room_id): min_depth = self._simple_select_one_onecol_txn( txn, From 946d02536be9e94888bdd399109c474b2cddeac2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 11:45:08 +0000 Subject: [PATCH 038/227] Remove unused functions. --- synapse/state.py | 90 ------------------------------------------------ 1 file changed, 90 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index c514b118ae..f7249705ce 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -102,35 +102,6 @@ class StateHandler(object): defer.returnValue(True) - @defer.inlineCallbacks - @log_function - def handle_new_state(self, new_pdu): - """ Apply conflict resolution to `new_pdu`. - - This should be called on every new state pdu, regardless of whether or - not there is a conflict. - - This function is safe against the race of it getting called with two - `PDU`s trying to update the same state. - """ - - # This needs to be done in a transaction. - - is_new = yield self._handle_new_state(new_pdu) - - logger.debug("is_new: %s %s %s", is_new, new_pdu.pdu_id, new_pdu.origin) - - if is_new: - yield self.store.update_current_state( - pdu_id=new_pdu.pdu_id, - origin=new_pdu.origin, - context=new_pdu.context, - pdu_type=new_pdu.pdu_type, - state_key=new_pdu.state_key - ) - - defer.returnValue(is_new) - @defer.inlineCallbacks @log_function def annotate_state_groups(self, event, old_state=None): @@ -267,67 +238,6 @@ class StateHandler(object): # event.sender) return event.power_level - @defer.inlineCallbacks - @log_function - def _handle_new_state(self, new_pdu): - tree, missing_branch = yield self.store.get_unresolved_state_tree( - new_pdu - ) - new_branch, current_branch = tree - - logger.debug( - "_handle_new_state new=%s, current=%s", - new_branch, current_branch - ) - - if missing_branch is not None: - # We're missing some PDUs. Fetch them. - # TODO (erikj): Limit this. - missing_prev = tree[missing_branch][-1] - - pdu_id = missing_prev.prev_state_id - origin = missing_prev.prev_state_origin - - is_missing = yield self.store.get_pdu(pdu_id, origin) is None - if not is_missing: - raise Exception("Conflict resolution failed") - - yield self._replication.get_pdu( - destination=missing_prev.origin, - pdu_origin=origin, - pdu_id=pdu_id, - outlier=True - ) - - updated_current = yield self._handle_new_state(new_pdu) - defer.returnValue(updated_current) - - if not current_branch: - # There is no current state - defer.returnValue(True) - return - - n = new_branch[-1] - c = current_branch[-1] - - common_ancestor = n.pdu_id == c.pdu_id and n.origin == c.origin - - if common_ancestor: - # We found a common ancestor! - - if len(current_branch) == 1: - # This is a direct clobber so we can just... - defer.returnValue(True) - - else: - # We didn't find a common ancestor. This is probably fine. - pass - - result = yield self._do_conflict_res( - new_branch, current_branch, common_ancestor - ) - defer.returnValue(result) - @defer.inlineCallbacks def _do_conflict_res(self, new_branch, current_branch, common_ancestor): conflict_res = [ From bfa36a72b9a852130cc42fb9322f6596e89725a7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 14:00:14 +0000 Subject: [PATCH 039/227] Remove PDU tables. --- synapse/federation/persistence.py | 70 --- synapse/federation/replication.py | 2 +- synapse/storage/__init__.py | 60 +- synapse/storage/pdu.py | 949 ------------------------------ synapse/storage/schema/pdu.sql | 106 ---- synapse/storage/transactions.py | 45 -- 6 files changed, 2 insertions(+), 1230 deletions(-) delete mode 100644 synapse/storage/pdu.py delete mode 100644 synapse/storage/schema/pdu.sql diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index 7043fcc504..a565375e68 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -32,76 +32,6 @@ import logging logger = logging.getLogger(__name__) -class PduActions(object): - """ Defines persistence actions that relate to handling PDUs. - """ - - def __init__(self, datastore): - self.store = datastore - - @log_function - def mark_as_processed(self, pdu): - """ Persist the fact that we have fully processed the given `Pdu` - - Returns: - Deferred - """ - return self.store.mark_pdu_as_processed(pdu.pdu_id, pdu.origin) - - @defer.inlineCallbacks - @log_function - def after_transaction(self, transaction_id, destination, origin): - """ Returns all `Pdu`s that we sent to the given remote home server - after a given transaction id. - - Returns: - Deferred: Results in a list of `Pdu`s - """ - results = yield self.store.get_pdus_after_transaction( - transaction_id, - destination - ) - - defer.returnValue([Pdu.from_pdu_tuple(p) for p in results]) - - @defer.inlineCallbacks - @log_function - def get_all_pdus_from_context(self, context): - results = yield self.store.get_all_pdus_from_context(context) - defer.returnValue([Pdu.from_pdu_tuple(p) for p in results]) - - @defer.inlineCallbacks - @log_function - def backfill(self, context, pdu_list, limit): - """ For a given list of PDU id and origins return the proceeding - `limit` `Pdu`s in the given `context`. - - Returns: - Deferred: Results in a list of `Pdu`s. - """ - results = yield self.store.get_backfill( - context, pdu_list, limit - ) - - defer.returnValue([Pdu.from_pdu_tuple(p) for p in results]) - - @log_function - def is_new(self, pdu): - """ When we receive a `Pdu` from a remote home server, we want to - figure out whether it is `new`, i.e. it is not some historic PDU that - we haven't seen simply because we haven't backfilled back that far. - - Returns: - Deferred: Results in a `bool` - """ - return self.store.is_pdu_new( - pdu_id=pdu.pdu_id, - origin=pdu.origin, - context=pdu.context, - depth=pdu.depth - ) - - class TransactionActions(object): """ Defines persistence actions that relate to handling Transactions. """ diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index a0bd2e0572..159af4eed7 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -21,7 +21,7 @@ from twisted.internet import defer from .units import Transaction, Pdu, Edu -from .persistence import PduActions, TransactionActions +from .persistence import TransactionActions from synapse.util.logutils import log_function diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index d75c366834..3faa571dd9 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -37,7 +37,6 @@ from .registration import RegistrationStore from .room import RoomStore from .roommember import RoomMemberStore from .stream import StreamStore -from .pdu import StatePduStore, PduStore, PdusTable from .transactions import TransactionStore from .keys import KeyStore from .event_federation import EventFederationStore @@ -60,7 +59,6 @@ logger = logging.getLogger(__name__) SCHEMAS = [ "transactions", - "pdu", "users", "profiles", "presence", @@ -89,7 +87,7 @@ class _RollbackButIsFineException(Exception): class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, - PresenceStore, PduStore, StatePduStore, TransactionStore, + PresenceStore, TransactionStore, DirectoryStore, KeyStore, StateStore, SignatureStore, EventFederationStore, ): @@ -150,68 +148,12 @@ class DataStore(RoomMemberStore, RoomStore, def _persist_pdu_event_txn(self, txn, pdu=None, event=None, backfilled=False, stream_ordering=None, is_new_state=True): - if pdu is not None: - self._persist_event_pdu_txn(txn, pdu) if event is not None: return self._persist_event_txn( txn, event, backfilled, stream_ordering, is_new_state=is_new_state, ) - def _persist_event_pdu_txn(self, txn, pdu): - cols = dict(pdu.__dict__) - unrec_keys = dict(pdu.unrecognized_keys) - del cols["hashes"] - del cols["signatures"] - del cols["content"] - del cols["prev_pdus"] - cols["content_json"] = json.dumps(pdu.content) - - unrec_keys.update({ - k: v for k, v in cols.items() - if k not in PdusTable.fields - }) - - cols["unrecognized_keys"] = json.dumps(unrec_keys) - - cols["ts"] = cols.pop("origin_server_ts") - - logger.debug("Persisting: %s", repr(cols)) - - for hash_alg, hash_base64 in pdu.hashes.items(): - hash_bytes = decode_base64(hash_base64) - self._store_pdu_content_hash_txn( - txn, pdu.pdu_id, pdu.origin, hash_alg, hash_bytes, - ) - - signatures = pdu.signatures.get(pdu.origin, {}) - - for key_id, signature_base64 in signatures.items(): - signature_bytes = decode_base64(signature_base64) - self._store_pdu_origin_signature_txn( - txn, pdu.pdu_id, pdu.origin, key_id, signature_bytes, - ) - - for prev_pdu_id, prev_origin, prev_hashes in pdu.prev_pdus: - for alg, hash_base64 in prev_hashes.items(): - hash_bytes = decode_base64(hash_base64) - self._store_prev_pdu_hash_txn( - txn, pdu.pdu_id, pdu.origin, prev_pdu_id, prev_origin, alg, - hash_bytes - ) - - (ref_alg, ref_hash_bytes) = compute_pdu_event_reference_hash(pdu) - self._store_pdu_reference_hash_txn( - txn, pdu.pdu_id, pdu.origin, ref_alg, ref_hash_bytes - ) - - if pdu.is_state: - self._persist_state_txn(txn, pdu.prev_pdus, cols) - else: - self._persist_pdu_txn(txn, pdu.prev_pdus, cols) - - self._update_min_depth_for_context_txn(txn, pdu.context, pdu.depth) - @log_function def _persist_event_txn(self, txn, event, backfilled, stream_ordering=None, is_new_state=True): diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py deleted file mode 100644 index 4a4341907b..0000000000 --- a/synapse/storage/pdu.py +++ /dev/null @@ -1,949 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.internet import defer - -from ._base import SQLBaseStore, Table, JoinHelper - -from synapse.federation.units import Pdu -from synapse.util.logutils import log_function - -from syutil.base64util import encode_base64 - -from collections import namedtuple - -import logging - - -logger = logging.getLogger(__name__) - - -class PduStore(SQLBaseStore): - """A collection of queries for handling PDUs. - """ - - def get_pdu(self, pdu_id, origin): - """Given a pdu_id and origin, get a PDU. - - Args: - txn - pdu_id (str) - origin (str) - - Returns: - PduTuple: If the pdu does not exist in the database, returns None - """ - - return self.runInteraction( - "get_pdu", self._get_pdu_tuple, pdu_id, origin - ) - - def _get_pdu_tuple(self, txn, pdu_id, origin): - res = self._get_pdu_tuples(txn, [(pdu_id, origin)]) - return res[0] if res else None - - def _get_pdu_tuples(self, txn, pdu_id_tuples): - results = [] - for pdu_id, origin in pdu_id_tuples: - txn.execute( - PduEdgesTable.select_statement("pdu_id = ? AND origin = ?"), - (pdu_id, origin) - ) - - edges = [ - (r.prev_pdu_id, r.prev_origin) - for r in PduEdgesTable.decode_results(txn.fetchall()) - ] - - edge_hashes = self._get_prev_pdu_hashes_txn(txn, pdu_id, origin) - - hashes = self._get_pdu_content_hashes_txn(txn, pdu_id, origin) - signatures = self._get_pdu_origin_signatures_txn( - txn, pdu_id, origin - ) - - query = ( - "SELECT %(fields)s FROM %(pdus)s as p " - "LEFT JOIN %(state)s as s " - "ON p.pdu_id = s.pdu_id AND p.origin = s.origin " - "WHERE p.pdu_id = ? AND p.origin = ? " - ) % { - "fields": _pdu_state_joiner.get_fields( - PdusTable="p", StatePdusTable="s"), - "pdus": PdusTable.table_name, - "state": StatePdusTable.table_name, - } - - txn.execute(query, (pdu_id, origin)) - - row = txn.fetchone() - if row: - results.append(PduTuple( - PduEntry(*row), edges, hashes, signatures, edge_hashes - )) - - return results - - def get_current_state_for_context(self, context): - """Get a list of PDUs that represent the current state for a given - context - - Args: - context (str) - - Returns: - list: A list of PduTuples - """ - - return self.runInteraction( - "get_current_state_for_context", - self._get_current_state_for_context, - context - ) - - def _get_current_state_for_context(self, txn, context): - query = ( - "SELECT pdu_id, origin FROM %s WHERE context = ?" - % CurrentStateTable.table_name - ) - - logger.debug("get_current_state %s, Args=%s", query, context) - txn.execute(query, (context,)) - - res = txn.fetchall() - - logger.debug("get_current_state %d results", len(res)) - - return self._get_pdu_tuples(txn, res) - - def _persist_pdu_txn(self, txn, prev_pdus, cols): - """Inserts a (non-state) PDU into the database. - - Args: - txn, - prev_pdus (list) - **cols: The columns to insert into the PdusTable. - """ - entry = PdusTable.EntryType( - **{k: cols.get(k, None) for k in PdusTable.fields} - ) - - txn.execute(PdusTable.insert_statement(), entry) - - self._handle_prev_pdus( - txn, entry.outlier, entry.pdu_id, entry.origin, - prev_pdus, entry.context - ) - - def mark_pdu_as_processed(self, pdu_id, pdu_origin): - """Mark a received PDU as processed. - - Args: - txn - pdu_id (str) - pdu_origin (str) - """ - - return self.runInteraction( - "mark_pdu_as_processed", - self._mark_as_processed, pdu_id, pdu_origin - ) - - def _mark_as_processed(self, txn, pdu_id, pdu_origin): - txn.execute("UPDATE %s SET have_processed = 1" % PdusTable.table_name) - - def get_all_pdus_from_context(self, context): - """Get a list of all PDUs for a given context.""" - return self.runInteraction( - "get_all_pdus_from_context", - self._get_all_pdus_from_context, context, - ) - - def _get_all_pdus_from_context(self, txn, context): - query = ( - "SELECT pdu_id, origin FROM %s " - "WHERE context = ?" - ) % PdusTable.table_name - - txn.execute(query, (context,)) - - return self._get_pdu_tuples(txn, txn.fetchall()) - - def get_backfill(self, context, pdu_list, limit): - """Get a list of Pdus for a given topic that occured before (and - including) the pdus in pdu_list. Return a list of max size `limit`. - - Args: - txn - context (str) - pdu_list (list) - limit (int) - - Return: - list: A list of PduTuples - """ - return self.runInteraction( - "get_backfill", - self._get_backfill, context, pdu_list, limit - ) - - def _get_backfill(self, txn, context, pdu_list, limit): - logger.debug( - "backfill: %s, %s, %s", - context, repr(pdu_list), limit - ) - - # We seed the pdu_results with the things from the pdu_list. - pdu_results = pdu_list - - front = pdu_list - - query = ( - "SELECT prev_pdu_id, prev_origin FROM %(edges_table)s " - "WHERE context = ? AND pdu_id = ? AND origin = ? " - "LIMIT ?" - ) % { - "edges_table": PduEdgesTable.table_name, - } - - # We iterate through all pdu_ids in `front` to select their previous - # pdus. These are dumped in `new_front`. We continue until we reach the - # limit *or* new_front is empty (i.e., we've run out of things to - # select - while front and len(pdu_results) < limit: - - new_front = [] - for pdu_id, origin in front: - logger.debug( - "_backfill_interaction: i=%s, o=%s", - pdu_id, origin - ) - - txn.execute( - query, - (context, pdu_id, origin, limit - len(pdu_results)) - ) - - for row in txn.fetchall(): - logger.debug( - "_backfill_interaction: got i=%s, o=%s", - *row - ) - new_front.append(row) - - front = new_front - pdu_results += new_front - - # We also want to update the `prev_pdus` attributes before returning. - return self._get_pdu_tuples(txn, pdu_results) - - def get_min_depth_for_context(self, context): - """Get the current minimum depth for a context - - Args: - txn - context (str) - """ - return self.runInteraction( - "get_min_depth_for_context", - self._get_min_depth_for_context, context - ) - - def _get_min_depth_for_context(self, txn, context): - return self._get_min_depth_interaction(txn, context) - - def _get_min_depth_interaction(self, txn, context): - txn.execute( - "SELECT min_depth FROM %s WHERE context = ?" - % ContextDepthTable.table_name, - (context,) - ) - - row = txn.fetchone() - - return row[0] if row else None - - def _update_min_depth_for_context_txn(self, txn, context, depth): - """Update the minimum `depth` of the given context, which is the line - on which we stop backfilling backwards. - - Args: - context (str) - depth (int) - """ - min_depth = self._get_min_depth_interaction(txn, context) - - do_insert = depth < min_depth if min_depth else True - - if do_insert: - txn.execute( - "INSERT OR REPLACE INTO %s (context, min_depth) " - "VALUES (?,?)" % ContextDepthTable.table_name, - (context, depth) - ) - - def get_latest_pdus_in_context(self, context): - return self.runInteraction( - "get_latest_pdus_in_context", - self._get_latest_pdus_in_context, - context - ) - - def _get_latest_pdus_in_context(self, txn, context): - """Get's a list of the most current pdus for a given context. This is - used when we are sending a Pdu and need to fill out the `prev_pdus` - key - - Args: - txn - context - """ - query = ( - "SELECT p.pdu_id, p.origin, p.depth FROM %(pdus)s as p " - "INNER JOIN %(forward)s as f ON p.pdu_id = f.pdu_id " - "AND f.origin = p.origin " - "WHERE f.context = ?" - ) % { - "pdus": PdusTable.table_name, - "forward": PduForwardExtremitiesTable.table_name, - } - - logger.debug("get_prev query: %s", query) - - txn.execute( - query, - (context, ) - ) - - results = [] - for pdu_id, origin, depth in txn.fetchall(): - hashes = self._get_pdu_reference_hashes_txn(txn, pdu_id, origin) - sha256_bytes = hashes["sha256"] - prev_hashes = {"sha256": encode_base64(sha256_bytes)} - results.append((pdu_id, origin, prev_hashes, depth)) - - return results - - @defer.inlineCallbacks - def get_oldest_pdus_in_context(self, context): - """Get a list of Pdus that we haven't backfilled beyond yet (and havent - seen). This list is used when we want to backfill backwards and is the - list we send to the remote server. - - Args: - txn - context (str) - - Returns: - list: A list of PduIdTuple. - """ - results = yield self._execute( - None, - "SELECT pdu_id, origin FROM %(back)s WHERE context = ?" - % {"back": PduBackwardExtremitiesTable.table_name, }, - context - ) - - defer.returnValue([PduIdTuple(i, o) for i, o in results]) - - def is_pdu_new(self, pdu_id, origin, context, depth): - """For a given Pdu, try and figure out if it's 'new', i.e., if it's - not something we got randomly from the past, for example when we - request the current state of the room that will probably return a bunch - of pdus from before we joined. - - Args: - txn - pdu_id (str) - origin (str) - context (str) - depth (int) - - Returns: - bool - """ - - return self.runInteraction( - "is_pdu_new", - self._is_pdu_new, - pdu_id=pdu_id, - origin=origin, - context=context, - depth=depth - ) - - def _is_pdu_new(self, txn, pdu_id, origin, context, depth): - # If depth > min depth in back table, then we classify it as new. - # OR if there is nothing in the back table, then it kinda needs to - # be a new thing. - query = ( - "SELECT min(p.depth) FROM %(edges)s as e " - "INNER JOIN %(back)s as b " - "ON e.prev_pdu_id = b.pdu_id AND e.prev_origin = b.origin " - "INNER JOIN %(pdus)s as p " - "ON e.pdu_id = p.pdu_id AND p.origin = e.origin " - "WHERE p.context = ?" - ) % { - "pdus": PdusTable.table_name, - "edges": PduEdgesTable.table_name, - "back": PduBackwardExtremitiesTable.table_name, - } - - txn.execute(query, (context,)) - - min_depth, = txn.fetchone() - - if not min_depth or depth > int(min_depth): - logger.debug( - "is_new true: id=%s, o=%s, d=%s min_depth=%s", - pdu_id, origin, depth, min_depth - ) - return True - - # If this pdu is in the forwards table, then it also is a new one - query = ( - "SELECT * FROM %(forward)s WHERE pdu_id = ? AND origin = ?" - ) % { - "forward": PduForwardExtremitiesTable.table_name, - } - - txn.execute(query, (pdu_id, origin)) - - # Did we get anything? - if txn.fetchall(): - logger.debug( - "is_new true: id=%s, o=%s, d=%s was forward", - pdu_id, origin, depth - ) - return True - - logger.debug( - "is_new false: id=%s, o=%s, d=%s", - pdu_id, origin, depth - ) - - # FINE THEN. It's probably old. - return False - - @staticmethod - @log_function - def _handle_prev_pdus(txn, outlier, pdu_id, origin, prev_pdus, - context): - txn.executemany( - PduEdgesTable.insert_statement(), - [(pdu_id, origin, p[0], p[1], context) for p in prev_pdus] - ) - - # Update the extremities table if this is not an outlier. - if not outlier: - - # First, we delete the new one from the forwards extremities table. - query = ( - "DELETE FROM %s WHERE pdu_id = ? AND origin = ?" - % PduForwardExtremitiesTable.table_name - ) - txn.executemany(query, list(p[:2] for p in prev_pdus)) - - # We only insert as a forward extremety the new pdu if there are no - # other pdus that reference it as a prev pdu - query = ( - "INSERT INTO %(table)s (pdu_id, origin, context) " - "SELECT ?, ?, ? WHERE NOT EXISTS (" - "SELECT 1 FROM %(pdu_edges)s WHERE " - "prev_pdu_id = ? AND prev_origin = ?" - ")" - ) % { - "table": PduForwardExtremitiesTable.table_name, - "pdu_edges": PduEdgesTable.table_name - } - - logger.debug("query: %s", query) - - txn.execute(query, (pdu_id, origin, context, pdu_id, origin)) - - # Insert all the prev_pdus as a backwards thing, they'll get - # deleted in a second if they're incorrect anyway. - txn.executemany( - PduBackwardExtremitiesTable.insert_statement(), - [(i, o, context) for i, o, _ in prev_pdus] - ) - - # Also delete from the backwards extremities table all ones that - # reference pdus that we have already seen - query = ( - "DELETE FROM %(pdu_back)s WHERE EXISTS (" - "SELECT 1 FROM %(pdus)s AS pdus " - "WHERE " - "%(pdu_back)s.pdu_id = pdus.pdu_id " - "AND %(pdu_back)s.origin = pdus.origin " - "AND not pdus.outlier " - ")" - ) % { - "pdu_back": PduBackwardExtremitiesTable.table_name, - "pdus": PdusTable.table_name, - } - txn.execute(query) - - -class StatePduStore(SQLBaseStore): - """A collection of queries for handling state PDUs. - """ - - def _persist_state_txn(self, txn, prev_pdus, cols): - """Inserts a state PDU into the database - - Args: - txn, - prev_pdus (list) - **cols: The columns to insert into the PdusTable and StatePdusTable - """ - pdu_entry = PdusTable.EntryType( - **{k: cols.get(k, None) for k in PdusTable.fields} - ) - state_entry = StatePdusTable.EntryType( - **{k: cols.get(k, None) for k in StatePdusTable.fields} - ) - - logger.debug("Inserting pdu: %s", repr(pdu_entry)) - logger.debug("Inserting state: %s", repr(state_entry)) - - txn.execute(PdusTable.insert_statement(), pdu_entry) - txn.execute(StatePdusTable.insert_statement(), state_entry) - - self._handle_prev_pdus( - txn, - pdu_entry.outlier, pdu_entry.pdu_id, pdu_entry.origin, prev_pdus, - pdu_entry.context - ) - - def get_unresolved_state_tree(self, new_state_pdu): - return self.runInteraction( - "get_unresolved_state_tree", - self._get_unresolved_state_tree, new_state_pdu - ) - - @log_function - def _get_unresolved_state_tree(self, txn, new_pdu): - current = self._get_current_interaction( - txn, - new_pdu.context, new_pdu.pdu_type, new_pdu.state_key - ) - - ReturnType = namedtuple( - "StateReturnType", ["new_branch", "current_branch"] - ) - return_value = ReturnType([new_pdu], []) - - if not current: - logger.debug("get_unresolved_state_tree No current state.") - return (return_value, None) - - return_value.current_branch.append(current) - - enum_branches = self._enumerate_state_branches( - txn, new_pdu, current - ) - - missing_branch = None - for branch, prev_state, state in enum_branches: - if state: - return_value[branch].append(state) - else: - # We don't have prev_state :( - missing_branch = branch - break - - return (return_value, missing_branch) - - def update_current_state(self, pdu_id, origin, context, pdu_type, - state_key): - return self.runInteraction( - "update_current_state", - self._update_current_state, - pdu_id, origin, context, pdu_type, state_key - ) - - def _update_current_state(self, txn, pdu_id, origin, context, pdu_type, - state_key): - query = ( - "INSERT OR REPLACE INTO %(curr)s (%(fields)s) VALUES (%(qs)s)" - ) % { - "curr": CurrentStateTable.table_name, - "fields": CurrentStateTable.get_fields_string(), - "qs": ", ".join(["?"] * len(CurrentStateTable.fields)) - } - - query_args = CurrentStateTable.EntryType( - pdu_id=pdu_id, - origin=origin, - context=context, - pdu_type=pdu_type, - state_key=state_key - ) - - txn.execute(query, query_args) - - def get_current_state_pdu(self, context, pdu_type, state_key): - """For a given context, pdu_type, state_key 3-tuple, return what is - currently considered the current state. - - Args: - txn - context (str) - pdu_type (str) - state_key (str) - - Returns: - PduEntry - """ - - return self.runInteraction( - "get_current_state_pdu", - self._get_current_state_pdu, context, pdu_type, state_key - ) - - def _get_current_state_pdu(self, txn, context, pdu_type, state_key): - return self._get_current_interaction(txn, context, pdu_type, state_key) - - def _get_current_interaction(self, txn, context, pdu_type, state_key): - logger.debug( - "_get_current_interaction %s %s %s", - context, pdu_type, state_key - ) - - fields = _pdu_state_joiner.get_fields( - PdusTable="p", StatePdusTable="s") - - current_query = ( - "SELECT %(fields)s FROM %(state)s as s " - "INNER JOIN %(pdus)s as p " - "ON s.pdu_id = p.pdu_id AND s.origin = p.origin " - "INNER JOIN %(curr)s as c " - "ON s.pdu_id = c.pdu_id AND s.origin = c.origin " - "WHERE s.context = ? AND s.pdu_type = ? AND s.state_key = ? " - ) % { - "fields": fields, - "curr": CurrentStateTable.table_name, - "state": StatePdusTable.table_name, - "pdus": PdusTable.table_name, - } - - txn.execute( - current_query, - (context, pdu_type, state_key) - ) - - row = txn.fetchone() - - result = PduEntry(*row) if row else None - - if not result: - logger.debug("_get_current_interaction not found") - else: - logger.debug( - "_get_current_interaction found %s %s", - result.pdu_id, result.origin - ) - - return result - - def handle_new_state(self, new_pdu): - """Actually perform conflict resolution on the new_pdu on the - assumption we have all the pdus required to perform it. - - Args: - new_pdu - - Returns: - bool: True if the new_pdu clobbered the current state, False if not - """ - return self.runInteraction( - "handle_new_state", - self._handle_new_state, new_pdu - ) - - def _handle_new_state(self, txn, new_pdu): - logger.debug( - "handle_new_state %s %s", - new_pdu.pdu_id, new_pdu.origin - ) - - current = self._get_current_interaction( - txn, - new_pdu.context, new_pdu.pdu_type, new_pdu.state_key - ) - - is_current = False - - if (not current or not current.prev_state_id - or not current.prev_state_origin): - # Oh, we don't have any state for this yet. - is_current = True - elif (current.pdu_id == new_pdu.prev_state_id - and current.origin == new_pdu.prev_state_origin): - # Oh! A direct clobber. Just do it. - is_current = True - else: - ## - # Ok, now loop through until we get to a common ancestor. - max_new = int(new_pdu.power_level) - max_current = int(current.power_level) - - enum_branches = self._enumerate_state_branches( - txn, new_pdu, current - ) - for branch, prev_state, state in enum_branches: - if not state: - raise RuntimeError( - "Could not find state_pdu %s %s" % - ( - prev_state.prev_state_id, - prev_state.prev_state_origin - ) - ) - - if branch == 0: - max_new = max(int(state.depth), max_new) - else: - max_current = max(int(state.depth), max_current) - - is_current = max_new > max_current - - if is_current: - logger.debug("handle_new_state make current") - - # Right, this is a new thing, so woo, just insert it. - txn.execute( - "INSERT OR REPLACE INTO %(curr)s (%(fields)s) VALUES (%(qs)s)" - % { - "curr": CurrentStateTable.table_name, - "fields": CurrentStateTable.get_fields_string(), - "qs": ", ".join(["?"] * len(CurrentStateTable.fields)) - }, - CurrentStateTable.EntryType( - *(new_pdu.__dict__[k] for k in CurrentStateTable.fields) - ) - ) - else: - logger.debug("handle_new_state not current") - - logger.debug("handle_new_state done") - - return is_current - - @log_function - def _enumerate_state_branches(self, txn, pdu_a, pdu_b): - branch_a = pdu_a - branch_b = pdu_b - - while True: - if (branch_a.pdu_id == branch_b.pdu_id - and branch_a.origin == branch_b.origin): - # Woo! We found a common ancestor - logger.debug("_enumerate_state_branches Found common ancestor") - break - - do_branch_a = ( - hasattr(branch_a, "prev_state_id") and - branch_a.prev_state_id - ) - - do_branch_b = ( - hasattr(branch_b, "prev_state_id") and - branch_b.prev_state_id - ) - - logger.debug( - "do_branch_a=%s, do_branch_b=%s", - do_branch_a, do_branch_b - ) - - if do_branch_a and do_branch_b: - do_branch_a = int(branch_a.depth) > int(branch_b.depth) - - if do_branch_a: - pdu_tuple = PduIdTuple( - branch_a.prev_state_id, - branch_a.prev_state_origin - ) - - prev_branch = branch_a - - logger.debug("getting branch_a prev %s", pdu_tuple) - branch_a = self._get_pdu_tuple(txn, *pdu_tuple) - if branch_a: - branch_a = Pdu.from_pdu_tuple(branch_a) - - logger.debug("branch_a=%s", branch_a) - - yield (0, prev_branch, branch_a) - - if not branch_a: - break - elif do_branch_b: - pdu_tuple = PduIdTuple( - branch_b.prev_state_id, - branch_b.prev_state_origin - ) - - prev_branch = branch_b - - logger.debug("getting branch_b prev %s", pdu_tuple) - branch_b = self._get_pdu_tuple(txn, *pdu_tuple) - if branch_b: - branch_b = Pdu.from_pdu_tuple(branch_b) - - logger.debug("branch_b=%s", branch_b) - - yield (1, prev_branch, branch_b) - - if not branch_b: - break - else: - break - - -class PdusTable(Table): - table_name = "pdus" - - fields = [ - "pdu_id", - "origin", - "context", - "pdu_type", - "ts", - "depth", - "is_state", - "content_json", - "unrecognized_keys", - "outlier", - "have_processed", - ] - - EntryType = namedtuple("PdusEntry", fields) - - -class PduDestinationsTable(Table): - table_name = "pdu_destinations" - - fields = [ - "pdu_id", - "origin", - "destination", - "delivered_ts", - ] - - EntryType = namedtuple("PduDestinationsEntry", fields) - - -class PduEdgesTable(Table): - table_name = "pdu_edges" - - fields = [ - "pdu_id", - "origin", - "prev_pdu_id", - "prev_origin", - "context" - ] - - EntryType = namedtuple("PduEdgesEntry", fields) - - -class PduForwardExtremitiesTable(Table): - table_name = "pdu_forward_extremities" - - fields = [ - "pdu_id", - "origin", - "context", - ] - - EntryType = namedtuple("PduForwardExtremitiesEntry", fields) - - -class PduBackwardExtremitiesTable(Table): - table_name = "pdu_backward_extremities" - - fields = [ - "pdu_id", - "origin", - "context", - ] - - EntryType = namedtuple("PduBackwardExtremitiesEntry", fields) - - -class ContextDepthTable(Table): - table_name = "context_depth" - - fields = [ - "context", - "min_depth", - ] - - EntryType = namedtuple("ContextDepthEntry", fields) - - -class StatePdusTable(Table): - table_name = "state_pdus" - - fields = [ - "pdu_id", - "origin", - "context", - "pdu_type", - "state_key", - "power_level", - "prev_state_id", - "prev_state_origin", - ] - - EntryType = namedtuple("StatePdusEntry", fields) - - -class CurrentStateTable(Table): - table_name = "current_state" - - fields = [ - "pdu_id", - "origin", - "context", - "pdu_type", - "state_key", - ] - - EntryType = namedtuple("CurrentStateEntry", fields) - -_pdu_state_joiner = JoinHelper(PdusTable, StatePdusTable) - - -# TODO: These should probably be put somewhere more sensible -PduIdTuple = namedtuple("PduIdTuple", ("pdu_id", "origin")) - -PduEntry = _pdu_state_joiner.EntryType -""" We are always interested in the join of the PdusTable and StatePdusTable, -rather than just the PdusTable. - -This does not include a prev_pdus key. -""" - -PduTuple = namedtuple( - "PduTuple", - ("pdu_entry", "prev_pdu_list", "hashes", "signatures", "edge_hashes") -) -""" This is a tuple of a `PduEntry` and a list of `PduIdTuple` that represent -the `prev_pdus` key of a PDU. -""" diff --git a/synapse/storage/schema/pdu.sql b/synapse/storage/schema/pdu.sql deleted file mode 100644 index 16e111a56c..0000000000 --- a/synapse/storage/schema/pdu.sql +++ /dev/null @@ -1,106 +0,0 @@ -/* Copyright 2014 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ --- Stores pdus and their content -CREATE TABLE IF NOT EXISTS pdus( - pdu_id TEXT, - origin TEXT, - context TEXT, - pdu_type TEXT, - ts INTEGER, - depth INTEGER DEFAULT 0 NOT NULL, - is_state BOOL, - content_json TEXT, - unrecognized_keys TEXT, - outlier BOOL NOT NULL, - have_processed BOOL, - CONSTRAINT pdu_id_origin UNIQUE (pdu_id, origin) -); - --- Stores what the current state pdu is for a given (context, pdu_type, key) tuple -CREATE TABLE IF NOT EXISTS state_pdus( - pdu_id TEXT, - origin TEXT, - context TEXT, - pdu_type TEXT, - state_key TEXT, - power_level TEXT, - prev_state_id TEXT, - prev_state_origin TEXT, - CONSTRAINT pdu_id_origin UNIQUE (pdu_id, origin) - CONSTRAINT prev_pdu_id_origin UNIQUE (prev_state_id, prev_state_origin) -); - -CREATE TABLE IF NOT EXISTS current_state( - pdu_id TEXT, - origin TEXT, - context TEXT, - pdu_type TEXT, - state_key TEXT, - CONSTRAINT pdu_id_origin UNIQUE (pdu_id, origin) - CONSTRAINT uniqueness UNIQUE (context, pdu_type, state_key) ON CONFLICT REPLACE -); - --- Stores where each pdu we want to send should be sent and the delivery status. -create TABLE IF NOT EXISTS pdu_destinations( - pdu_id TEXT, - origin TEXT, - destination TEXT, - delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered - CONSTRAINT uniqueness UNIQUE (pdu_id, origin, destination) ON CONFLICT REPLACE -); - -CREATE TABLE IF NOT EXISTS pdu_forward_extremities( - pdu_id TEXT, - origin TEXT, - context TEXT, - CONSTRAINT uniqueness UNIQUE (pdu_id, origin, context) ON CONFLICT REPLACE -); - -CREATE TABLE IF NOT EXISTS pdu_backward_extremities( - pdu_id TEXT, - origin TEXT, - context TEXT, - CONSTRAINT uniqueness UNIQUE (pdu_id, origin, context) ON CONFLICT REPLACE -); - -CREATE TABLE IF NOT EXISTS pdu_edges( - pdu_id TEXT, - origin TEXT, - prev_pdu_id TEXT, - prev_origin TEXT, - context TEXT, - CONSTRAINT uniqueness UNIQUE (pdu_id, origin, prev_pdu_id, prev_origin, context) -); - -CREATE TABLE IF NOT EXISTS context_depth( - context TEXT, - min_depth INTEGER, - CONSTRAINT uniqueness UNIQUE (context) -); - -CREATE INDEX IF NOT EXISTS context_depth_context ON context_depth(context); - - -CREATE INDEX IF NOT EXISTS pdu_id ON pdus(pdu_id, origin); - -CREATE INDEX IF NOT EXISTS dests_id ON pdu_destinations (pdu_id, origin); --- CREATE INDEX IF NOT EXISTS dests ON pdu_destinations (destination); - -CREATE INDEX IF NOT EXISTS pdu_extrem_context ON pdu_forward_extremities(context); -CREATE INDEX IF NOT EXISTS pdu_extrem_id ON pdu_forward_extremities(pdu_id, origin); - -CREATE INDEX IF NOT EXISTS pdu_edges_id ON pdu_edges(pdu_id, origin); - -CREATE INDEX IF NOT EXISTS pdu_b_extrem_context ON pdu_backward_extremities(context); diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 6624348fd0..ea67900788 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -14,7 +14,6 @@ # limitations under the License. from ._base import SQLBaseStore, Table -from .pdu import PdusTable from collections import namedtuple @@ -207,50 +206,6 @@ class TransactionStore(SQLBaseStore): return ReceivedTransactionsTable.decode_results(txn.fetchall()) - def get_pdus_after_transaction(self, transaction_id, destination): - """For a given local transaction_id that we sent to a given destination - home server, return a list of PDUs that were sent to that destination - after it. - - Args: - txn - transaction_id (str) - destination (str) - - Returns - list: A list of PduTuple - """ - return self.runInteraction( - "get_pdus_after_transaction", - self._get_pdus_after_transaction, - transaction_id, destination - ) - - def _get_pdus_after_transaction(self, txn, transaction_id, destination): - - # Query that first get's all transaction_ids with an id greater than - # the one given from the `sent_transactions` table. Then JOIN on this - # from the `tx->pdu` table to get a list of (pdu_id, origin) that - # specify the pdus that were sent in those transactions. - query = ( - "SELECT pdu_id, pdu_origin FROM %(tx_pdu)s as tp " - "INNER JOIN %(sent_tx)s as st " - "ON tp.transaction_id = st.transaction_id " - "AND tp.destination = st.destination " - "WHERE st.id > (" - "SELECT id FROM %(sent_tx)s " - "WHERE transaction_id = ? AND destination = ?" - ) % { - "tx_pdu": TransactionsToPduTable.table_name, - "sent_tx": SentTransactions.table_name, - } - - txn.execute(query, (transaction_id, destination)) - - pdus = PdusTable.decode_results(txn.fetchall()) - - return self._get_pdu_tuples(txn, pdus) - class ReceivedTransactionsTable(Table): table_name = "received_transactions" From 2f39dc19a26cca25305d10654916d7413a56a23a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 14:27:14 +0000 Subject: [PATCH 040/227] Remove more references to dead PDU tables --- synapse/federation/pdu_codec.py | 4 +- synapse/state.py | 23 +++------- synapse/storage/__init__.py | 9 ---- synapse/storage/schema/signatures.sql | 66 --------------------------- 4 files changed, 8 insertions(+), 94 deletions(-) delete mode 100644 synapse/storage/schema/signatures.sql diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index 6d31286290..d4c896e163 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -32,11 +32,11 @@ class PduCodec(object): self.hs = hs def encode_event_id(self, local, domain): - return EventID.create(local, domain, self.hs).to_string() + return local def decode_event_id(self, event_id): e_id = self.hs.parse_eventid(event_id) - return e_id.localpart, e_id.domain + return event_id, e_id.domain def event_from_pdu(self, pdu): kwargs = {} diff --git a/synapse/state.py b/synapse/state.py index f7249705ce..2548deed28 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -77,29 +77,18 @@ class StateHandler(object): snapshot.fill_out_prev_events(event) yield self.annotate_state_groups(event) - current_state = snapshot.prev_state_pdu + if event.old_state_events: + current_state = event.old_state_events.get( + (event.type, event.state_key) + ) - if current_state: - event.prev_state = EventID.create( - current_state.pdu_id, current_state.origin, self.hs - ).to_string() + if current_state: + event.prev_state = current_state.event_id # TODO check current_state to see if the min power level is less # than the power level of the user # power_level = self._get_power_level_for_event(event) - e_id = self.hs.parse_eventid(event.event_id) - pdu_id = e_id.localpart - origin = e_id.domain - - yield self.store.update_current_state( - pdu_id=pdu_id, - origin=origin, - context=key.context, - pdu_type=key.type, - state_key=key.state_key - ) - defer.returnValue(True) @defer.inlineCallbacks diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 3faa571dd9..c2560f6045 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -67,7 +67,6 @@ SCHEMAS = [ "keys", "redactions", "state", - "signatures", "event_edges", "event_signatures", ] @@ -364,13 +363,6 @@ class DataStore(RoomMemberStore, RoomStore, membership_state = self._get_room_member(txn, user_id, room_id) prev_events = self._get_latest_events_in_room(txn, room_id) - if state_type is not None and state_key is not None: - prev_state_pdu = self._get_current_state_pdu( - txn, room_id, state_type, state_key - ) - else: - prev_state_pdu = None - return Snapshot( store=self, room_id=room_id, @@ -379,7 +371,6 @@ class DataStore(RoomMemberStore, RoomStore, membership_state=membership_state, state_type=state_type, state_key=state_key, - prev_state_pdu=prev_state_pdu, ) return self.runInteraction("snapshot_room", _snapshot) diff --git a/synapse/storage/schema/signatures.sql b/synapse/storage/schema/signatures.sql deleted file mode 100644 index 1c45a51bec..0000000000 --- a/synapse/storage/schema/signatures.sql +++ /dev/null @@ -1,66 +0,0 @@ -/* Copyright 2014 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS pdu_content_hashes ( - pdu_id TEXT, - origin TEXT, - algorithm TEXT, - hash BLOB, - CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm) -); - -CREATE INDEX IF NOT EXISTS pdu_content_hashes_id ON pdu_content_hashes ( - pdu_id, origin -); - -CREATE TABLE IF NOT EXISTS pdu_reference_hashes ( - pdu_id TEXT, - origin TEXT, - algorithm TEXT, - hash BLOB, - CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm) -); - -CREATE INDEX IF NOT EXISTS pdu_reference_hashes_id ON pdu_reference_hashes ( - pdu_id, origin -); - -CREATE TABLE IF NOT EXISTS pdu_origin_signatures ( - pdu_id TEXT, - origin TEXT, - key_id TEXT, - signature BLOB, - CONSTRAINT uniqueness UNIQUE (pdu_id, origin, key_id) -); - -CREATE INDEX IF NOT EXISTS pdu_origin_signatures_id ON pdu_origin_signatures ( - pdu_id, origin -); - -CREATE TABLE IF NOT EXISTS pdu_edge_hashes( - pdu_id TEXT, - origin TEXT, - prev_pdu_id TEXT, - prev_origin TEXT, - algorithm TEXT, - hash BLOB, - CONSTRAINT uniqueness UNIQUE ( - pdu_id, origin, prev_pdu_id, prev_origin, algorithm - ) -); - -CREATE INDEX IF NOT EXISTS pdu_edge_hashes_id ON pdu_edge_hashes( - pdu_id, origin -); From d30d79b5bed98c7e46852c54875c976d3ac3bc0c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 15:35:39 +0000 Subject: [PATCH 041/227] Make prev_event signing work again. --- synapse/crypto/event_signing.py | 13 ++++++++++++- synapse/storage/__init__.py | 11 +++++------ synapse/storage/event_federation.py | 2 +- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 61edd2c6f9..07e383e221 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -16,11 +16,12 @@ from synapse.federation.units import Pdu -from synapse.api.events.utils import prune_pdu +from synapse.api.events.utils import prune_pdu, prune_event from syutil.jsonutil import encode_canonical_json from syutil.base64util import encode_base64, decode_base64 from syutil.crypto.jsonsign import sign_json, verify_signed_json +import copy import hashlib import logging @@ -69,6 +70,16 @@ def compute_pdu_event_reference_hash(pdu, hash_algorithm=hashlib.sha256): return (hashed.name, hashed.digest()) +def compute_event_reference_hash(event, hash_algorithm=hashlib.sha256): + tmp_event = copy.deepcopy(event) + tmp_event = prune_event(tmp_event) + event_json = tmp_event.get_dict() + event_json.pop("signatures", None) + event_json_bytes = encode_canonical_json(event_json) + hashed = hash_algorithm(event_json_bytes) + return (hashed.name, hashed.digest()) + + def sign_event_pdu(pdu, signature_name, signing_key): tmp_pdu = Pdu(**pdu.get_dict()) tmp_pdu = prune_pdu(tmp_pdu) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index c2560f6045..31a0022d54 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -46,7 +46,7 @@ from .signatures import SignatureStore from syutil.base64util import decode_base64 -from synapse.crypto.event_signing import compute_pdu_event_reference_hash +from synapse.crypto.event_signing import compute_event_reference_hash import json @@ -271,11 +271,10 @@ class DataStore(RoomMemberStore, RoomStore, txn, event.event_id, prev_event_id, alg, hash_bytes ) - # TODO - # (ref_alg, ref_hash_bytes) = compute_pdu_event_reference_hash(pdu) - # self._store_event_reference_hash_txn( - # txn, event.event_id, ref_alg, ref_hash_bytes - # ) + (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) + self._store_event_reference_hash_txn( + txn, event.event_id, ref_alg, ref_hash_bytes + ) self._update_min_depth_for_room_txn(txn, event.room_id, event.depth) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 8357071db6..dcc116bad2 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -69,7 +69,7 @@ class EventFederationStore(SQLBaseStore): results = [] for event_id, depth in txn.fetchall(): - hashes = self._get_prev_event_hashes_txn(txn, event_id) + hashes = self._get_event_reference_hashes_txn(txn, event_id) prev_hashes = { k: encode_base64(v) for k, v in hashes.items() if k == "sha256" From fb3a01fa3a2603751ac427f90597d921c8662417 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 16:04:27 +0000 Subject: [PATCH 042/227] Remove unused sql file. --- synapse/storage/schema/edge_pdus.sql | 31 ---------------------------- 1 file changed, 31 deletions(-) delete mode 100644 synapse/storage/schema/edge_pdus.sql diff --git a/synapse/storage/schema/edge_pdus.sql b/synapse/storage/schema/edge_pdus.sql deleted file mode 100644 index 8a00868065..0000000000 --- a/synapse/storage/schema/edge_pdus.sql +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright 2014 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS context_edge_pdus( - id INTEGER PRIMARY KEY AUTOINCREMENT, -- twistar requires this - pdu_id TEXT, - origin TEXT, - context TEXT, - CONSTRAINT context_edge_pdu_id_origin UNIQUE (pdu_id, origin) -); - -CREATE TABLE IF NOT EXISTS origin_edge_pdus( - id INTEGER PRIMARY KEY AUTOINCREMENT, -- twistar requires this - pdu_id TEXT, - origin TEXT, - CONSTRAINT origin_edge_pdu_id_origin UNIQUE (pdu_id, origin) -); - -CREATE INDEX IF NOT EXISTS context_edge_pdu_id ON context_edge_pdus(pdu_id, origin); -CREATE INDEX IF NOT EXISTS origin_edge_pdu_id ON origin_edge_pdus(pdu_id, origin); From 80b2710e6f9b7520f49f82b117634294c63fdb1e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 17:08:36 +0000 Subject: [PATCH 043/227] Remove unused signature storage methods --- synapse/storage/signatures.py | 139 +--------------------------------- 1 file changed, 1 insertion(+), 138 deletions(-) diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 5e99174fcd..b4b3d5d7ea 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -17,144 +17,7 @@ from _base import SQLBaseStore class SignatureStore(SQLBaseStore): - """Persistence for PDU signatures and hashes""" - - def _get_pdu_content_hashes_txn(self, txn, pdu_id, origin): - """Get all the hashes for a given PDU. - Args: - txn (cursor): - pdu_id (str): Id for the PDU. - origin (str): origin of the PDU. - Returns: - A dict of algorithm -> hash. - """ - query = ( - "SELECT algorithm, hash" - " FROM pdu_content_hashes" - " WHERE pdu_id = ? and origin = ?" - ) - txn.execute(query, (pdu_id, origin)) - return dict(txn.fetchall()) - - def _store_pdu_content_hash_txn(self, txn, pdu_id, origin, algorithm, - hash_bytes): - """Store a hash for a PDU - Args: - txn (cursor): - pdu_id (str): Id for the PDU. - origin (str): origin of the PDU. - algorithm (str): Hashing algorithm. - hash_bytes (bytes): Hash function output bytes. - """ - self._simple_insert_txn(txn, "pdu_content_hashes", { - "pdu_id": pdu_id, - "origin": origin, - "algorithm": algorithm, - "hash": buffer(hash_bytes), - }) - - def _get_pdu_reference_hashes_txn(self, txn, pdu_id, origin): - """Get all the hashes for a given PDU. - Args: - txn (cursor): - pdu_id (str): Id for the PDU. - origin (str): origin of the PDU. - Returns: - A dict of algorithm -> hash. - """ - query = ( - "SELECT algorithm, hash" - " FROM pdu_reference_hashes" - " WHERE pdu_id = ? and origin = ?" - ) - txn.execute(query, (pdu_id, origin)) - return dict(txn.fetchall()) - - def _store_pdu_reference_hash_txn(self, txn, pdu_id, origin, algorithm, - hash_bytes): - """Store a hash for a PDU - Args: - txn (cursor): - pdu_id (str): Id for the PDU. - origin (str): origin of the PDU. - algorithm (str): Hashing algorithm. - hash_bytes (bytes): Hash function output bytes. - """ - self._simple_insert_txn(txn, "pdu_reference_hashes", { - "pdu_id": pdu_id, - "origin": origin, - "algorithm": algorithm, - "hash": buffer(hash_bytes), - }) - - - def _get_pdu_origin_signatures_txn(self, txn, pdu_id, origin): - """Get all the signatures for a given PDU. - Args: - txn (cursor): - pdu_id (str): Id for the PDU. - origin (str): origin of the PDU. - Returns: - A dict of key_id -> signature_bytes. - """ - query = ( - "SELECT key_id, signature" - " FROM pdu_origin_signatures" - " WHERE pdu_id = ? and origin = ?" - ) - txn.execute(query, (pdu_id, origin)) - return dict(txn.fetchall()) - - def _store_pdu_origin_signature_txn(self, txn, pdu_id, origin, key_id, - signature_bytes): - """Store a signature from the origin server for a PDU. - Args: - txn (cursor): - pdu_id (str): Id for the PDU. - origin (str): origin of the PDU. - key_id (str): Id for the signing key. - signature (bytes): The signature. - """ - self._simple_insert_txn(txn, "pdu_origin_signatures", { - "pdu_id": pdu_id, - "origin": origin, - "key_id": key_id, - "signature": buffer(signature_bytes), - }) - - def _get_prev_pdu_hashes_txn(self, txn, pdu_id, origin): - """Get all the hashes for previous PDUs of a PDU - Args: - txn (cursor): - pdu_id (str): Id of the PDU. - origin (str): Origin of the PDU. - Returns: - dict of (pdu_id, origin) -> dict of algorithm -> hash_bytes. - """ - query = ( - "SELECT prev_pdu_id, prev_origin, algorithm, hash" - " FROM pdu_edge_hashes" - " WHERE pdu_id = ? and origin = ?" - ) - txn.execute(query, (pdu_id, origin)) - results = {} - for prev_pdu_id, prev_origin, algorithm, hash_bytes in txn.fetchall(): - hashes = results.setdefault((prev_pdu_id, prev_origin), {}) - hashes[algorithm] = hash_bytes - return results - - def _store_prev_pdu_hash_txn(self, txn, pdu_id, origin, prev_pdu_id, - prev_origin, algorithm, hash_bytes): - self._simple_insert_txn(txn, "pdu_edge_hashes", { - "pdu_id": pdu_id, - "origin": origin, - "prev_pdu_id": prev_pdu_id, - "prev_origin": prev_origin, - "algorithm": algorithm, - "hash": buffer(hash_bytes), - }) - - ## Events ## + """Persistence for event signatures and hashes""" def _get_event_content_hashes_txn(self, txn, event_id): """Get all the hashes for a given Event. From ecabff7eb49ea799d9f52fad1e05f1f9a4b31e1c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 17:08:52 +0000 Subject: [PATCH 044/227] Sign evnets --- synapse/crypto/event_signing.py | 20 ++++++++++++++++++++ synapse/storage/__init__.py | 6 ++++++ 2 files changed, 26 insertions(+) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 07e383e221..cb2db01c04 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -94,3 +94,23 @@ def verify_signed_event_pdu(pdu, signature_name, verify_key): tmp_pdu = prune_pdu(tmp_pdu) pdu_json = tmp_pdu.get_dict() verify_signed_json(pdu_json, signature_name, verify_key) + + +def add_hashes_and_signatures(event, signature_name, signing_key, + hash_algorithm=hashlib.sha256): + tmp_event = copy.deepcopy(event) + tmp_event = prune_event(tmp_event) + redact_json = tmp_event.get_dict() + redact_json.pop("signatures", None) + redact_json = sign_json(redact_json, signature_name, signing_key) + event.signatures = redact_json["signatures"] + + event_json = event.get_full_dict() + #TODO: We need to sign the JSON that is going out via fedaration. + event_json.pop("age_ts", None) + event_json.pop("unsigned", None) + event_json.pop("signatures", None) + event_json.pop("hashes", None) + event_json_bytes = encode_canonical_json(event_json) + hashed = hash_algorithm(event_json_bytes) + event.hashes[hashed.name] = encode_base64(hashed.digest()) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 31a0022d54..1f39a4094e 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -255,6 +255,12 @@ class DataStore(RoomMemberStore, RoomStore, } ) + for hash_alg, hash_base64 in event.hashes.items(): + hash_bytes = decode_base64(hash_base64) + self._store_event_content_hash_txn( + txn, event.event_id, hash_alg, hash_bytes, + ) + if hasattr(event, "signatures"): signatures = event.signatures.get(event.origin, {}) From 9024a196583d6abf7a23ea3c7a90501f0d9dad16 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 11:31:47 +0000 Subject: [PATCH 045/227] Remove dead code. --- synapse/state.py | 84 ------------------------------------------------ 1 file changed, 84 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 2548deed28..f4efc287c9 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -65,12 +65,6 @@ class StateHandler(object): if not hasattr(event, "state_key"): return - key = KeyStateTuple( - event.room_id, - event.type, - _get_state_key_from_event(event) - ) - # Now I need to fill out the prev state and work out if it has auth # (w.r.t. to power levels) @@ -85,10 +79,6 @@ class StateHandler(object): if current_state: event.prev_state = current_state.event_id - # TODO check current_state to see if the min power level is less - # than the power level of the user - # power_level = self._get_power_level_for_event(event) - defer.returnValue(True) @defer.inlineCallbacks @@ -221,77 +211,3 @@ class StateHandler(object): ).hexdigest() )[0] ) - - def _get_power_level_for_event(self, event): - # return self._persistence.get_power_level_for_user(event.room_id, - # event.sender) - return event.power_level - - @defer.inlineCallbacks - def _do_conflict_res(self, new_branch, current_branch, common_ancestor): - conflict_res = [ - self._do_power_level_conflict_res, - self._do_chain_length_conflict_res, - self._do_hash_conflict_res, - ] - - for algo in conflict_res: - new_res, curr_res = yield defer.maybeDeferred( - algo, - new_branch, current_branch, common_ancestor - ) - - if new_res < curr_res: - defer.returnValue(False) - elif new_res > curr_res: - defer.returnValue(True) - - raise Exception("Conflict resolution failed.") - - @defer.inlineCallbacks - def _do_power_level_conflict_res(self, new_branch, current_branch, - common_ancestor): - new_powers_deferreds = [] - for e in new_branch[:-1] if common_ancestor else new_branch: - if hasattr(e, "user_id"): - new_powers_deferreds.append( - self.store.get_power_level(e.context, e.user_id) - ) - - current_powers_deferreds = [] - for e in current_branch[:-1] if common_ancestor else current_branch: - if hasattr(e, "user_id"): - current_powers_deferreds.append( - self.store.get_power_level(e.context, e.user_id) - ) - - new_powers = yield defer.gatherResults( - new_powers_deferreds, - consumeErrors=True - ) - - current_powers = yield defer.gatherResults( - current_powers_deferreds, - consumeErrors=True - ) - - max_power_new = max(new_powers) - max_power_current = max(current_powers) - - defer.returnValue( - (max_power_new, max_power_current) - ) - - def _do_chain_length_conflict_res(self, new_branch, current_branch, - common_ancestor): - return (len(new_branch), len(current_branch)) - - def _do_hash_conflict_res(self, new_branch, current_branch, - common_ancestor): - new_str = "".join([p.pdu_id + p.origin for p in new_branch]) - c_str = "".join([p.pdu_id + p.origin for p in current_branch]) - - return ( - hashlib.sha1(new_str).hexdigest(), - hashlib.sha1(c_str).hexdigest() - ) From 82a6b83524ed2f7cb12bd2fc43a2651558c392dd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 11:32:12 +0000 Subject: [PATCH 046/227] Don't assume event has hashes key already --- synapse/crypto/event_signing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index cb2db01c04..0e8bc7eb6c 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -113,4 +113,6 @@ def add_hashes_and_signatures(event, signature_name, signing_key, event_json.pop("hashes", None) event_json_bytes = encode_canonical_json(event_json) hashed = hash_algorithm(event_json_bytes) + if not hasattr(event, "hashes"): + event.hashes = {} event.hashes[hashed.name] = encode_base64(hashed.digest()) From 0a8b026ccf2d2472bb14303eaf5af8b8cf0efa5d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 11:32:42 +0000 Subject: [PATCH 047/227] Add 'origin' key to events --- synapse/api/events/factory.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index 750096c618..9134c82eff 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -69,6 +69,10 @@ class EventFactory(object): kwargs["type"] = etype if "event_id" not in kwargs: kwargs["event_id"] = self.create_event_id() + kwargs["origin"] = self.hs.hostname + else: + ev_id = self.hs.parse_eventid(kwargs["event_id"]) + kwargs["origin"] = ev_id.domain if "origin_server_ts" not in kwargs: kwargs["origin_server_ts"] = int(self.clock.time_msec()) From 7249785bcbe86a4e1c7f6b4f4f5b0601f8a8c47a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 11:33:28 +0000 Subject: [PATCH 048/227] Sign events we create. --- synapse/handlers/_base.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 787a01efc5..28b64565ae 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -18,6 +18,8 @@ from synapse.api.errors import LimitExceededError from synapse.util.async import run_on_reactor +from synapse.crypto.event_signing import add_hashes_and_signatures + class BaseHandler(object): def __init__(self, hs): @@ -32,6 +34,9 @@ class BaseHandler(object): self.clock = hs.get_clock() self.hs = hs + self.signing_key = hs.config.signing_key[0] + self.server_name = hs.hostname + def ratelimit(self, user_id): time_now = self.clock.time() allowed, time_allowed = self.ratelimiter.send_message( @@ -53,6 +58,10 @@ class BaseHandler(object): yield self.state_handler.annotate_state_groups(event) + yield add_hashes_and_signatures( + event, self.server_name, self.signing_key + ) + if not suppress_auth: yield self.auth.check(event, snapshot, raises=True) From f139c02e95e55f793c86ae6b7ad079d93aae0754 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 11:34:49 +0000 Subject: [PATCH 049/227] Formatting --- synapse/storage/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 1f39a4094e..6b8fed4502 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -199,7 +199,10 @@ class DataStore(RoomMemberStore, RoomStore, k: v for k, v in event.get_full_dict().items() if k not in vals.keys() and k not in [ - "redacted", "redacted_because", "signatures", "hashes", + "redacted", + "redacted_because", + "signatures", + "hashes", "prev_events", ] } From d59aa6af25472a15e6258f2fd9cbd1ac4c05702d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 11:35:19 +0000 Subject: [PATCH 050/227] For now, don't store txn -> pdu mappings. --- synapse/federation/persistence.py | 1 - synapse/storage/transactions.py | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index a565375e68..b04fbb4177 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -88,7 +88,6 @@ class TransactionActions(object): transaction.transaction_id, transaction.destination, transaction.origin_server_ts, - [(p["pdu_id"], p["origin"]) for p in transaction.pdus] ) @log_function diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index ea67900788..00d0f48082 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -89,7 +89,7 @@ class TransactionStore(SQLBaseStore): txn.execute(query, (code, response_json, transaction_id, origin)) def prep_send_transaction(self, transaction_id, destination, - origin_server_ts, pdu_list): + origin_server_ts): """Persists an outgoing transaction and calculates the values for the previous transaction id list. @@ -100,7 +100,6 @@ class TransactionStore(SQLBaseStore): transaction_id (str) destination (str) origin_server_ts (int) - pdu_list (list) Returns: list: A list of previous transaction ids. @@ -109,11 +108,11 @@ class TransactionStore(SQLBaseStore): return self.runInteraction( "prep_send_transaction", self._prep_send_transaction, - transaction_id, destination, origin_server_ts, pdu_list + transaction_id, destination, origin_server_ts ) def _prep_send_transaction(self, txn, transaction_id, destination, - origin_server_ts, pdu_list): + origin_server_ts): # First we find out what the prev_txs should be. # Since we know that we are only sending one transaction at a time, From ad6eacb3e9424902da9f83c8f106a4f0169c3108 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 13:06:58 +0000 Subject: [PATCH 051/227] Rename PDU fields to match that of events. --- synapse/api/events/utils.py | 2 +- synapse/federation/pdu_codec.py | 48 +------- synapse/federation/replication.py | 72 +++++------- synapse/federation/transport.py | 184 +++++------------------------- synapse/federation/units.py | 78 ++----------- synapse/handlers/federation.py | 12 +- 6 files changed, 80 insertions(+), 316 deletions(-) diff --git a/synapse/api/events/utils.py b/synapse/api/events/utils.py index 7fdf45a264..31601fd3a9 100644 --- a/synapse/api/events/utils.py +++ b/synapse/api/events/utils.py @@ -32,7 +32,7 @@ def prune_event(event): def prune_pdu(pdu): """Removes keys that contain unrestricted and non-essential data from a PDU """ - return _prune_event_or_pdu(pdu.pdu_type, pdu) + return _prune_event_or_pdu(pdu.type, pdu) def _prune_event_or_pdu(event_type, event): # Remove all extraneous fields. diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index d4c896e163..5ec97a698e 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -31,39 +31,16 @@ class PduCodec(object): self.clock = hs.get_clock() self.hs = hs - def encode_event_id(self, local, domain): - return local - - def decode_event_id(self, event_id): - e_id = self.hs.parse_eventid(event_id) - return event_id, e_id.domain - def event_from_pdu(self, pdu): kwargs = {} - kwargs["event_id"] = self.encode_event_id(pdu.pdu_id, pdu.origin) - kwargs["room_id"] = pdu.context - kwargs["etype"] = pdu.pdu_type - kwargs["prev_events"] = [ - (self.encode_event_id(i, o), s) - for i, o, s in pdu.prev_pdus - ] - - if hasattr(pdu, "prev_state_id") and hasattr(pdu, "prev_state_origin"): - kwargs["prev_state"] = self.encode_event_id( - pdu.prev_state_id, pdu.prev_state_origin - ) + kwargs["etype"] = pdu.type kwargs.update({ k: v for k, v in pdu.get_full_dict().items() if k not in [ - "pdu_id", - "context", - "pdu_type", - "prev_pdus", - "prev_state_id", - "prev_state_origin", + "type", ] }) @@ -72,33 +49,12 @@ class PduCodec(object): def pdu_from_event(self, event): d = event.get_full_dict() - d["pdu_id"], d["origin"] = self.decode_event_id( - event.event_id - ) - d["context"] = event.room_id - d["pdu_type"] = event.type - - if hasattr(event, "prev_events"): - def f(e, s): - i, o = self.decode_event_id(e) - return i, o, s - d["prev_pdus"] = [ - f(e, s) - for e, s in event.prev_events - ] - - if hasattr(event, "prev_state"): - d["prev_state_id"], d["prev_state_origin"] = ( - self.decode_event_id(event.prev_state) - ) - if hasattr(event, "state_key"): d["is_state"] = True kwargs = copy.deepcopy(event.unrecognized_keys) kwargs.update({ k: v for k, v in d.items() - if k not in ["event_id", "room_id", "type", "prev_events"] }) if "origin_server_ts" not in kwargs: diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 159af4eed7..838e660a46 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -111,14 +111,6 @@ class ReplicationLayer(object): """Informs the replication layer about a new PDU generated within the home server that should be transmitted to others. - This will fill out various attributes on the PDU object, e.g. the - `prev_pdus` key. - - *Note:* The home server should always call `send_pdu` even if it knows - that it does not need to be replicated to other home servers. This is - in case e.g. someone else joins via a remote home server and then - backfills. - TODO: Figure out when we should actually resolve the deferred. Args: @@ -131,18 +123,12 @@ class ReplicationLayer(object): order = self._order self._order += 1 - logger.debug("[%s] Persisting PDU", pdu.pdu_id) - - # Save *before* trying to send - # yield self.store.persist_event(pdu=pdu) - - logger.debug("[%s] Persisted PDU", pdu.pdu_id) - logger.debug("[%s] transaction_layer.enqueue_pdu... ", pdu.pdu_id) + logger.debug("[%s] transaction_layer.enqueue_pdu... ", pdu.event_id) # TODO, add errback, etc. self._transaction_queue.enqueue_pdu(pdu, order) - logger.debug("[%s] transaction_layer.enqueue_pdu... done", pdu.pdu_id) + logger.debug("[%s] transaction_layer.enqueue_pdu... done", pdu.event_id) @log_function def send_edu(self, destination, edu_type, content): @@ -215,7 +201,7 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def get_pdu(self, destination, pdu_origin, pdu_id, outlier=False): + def get_pdu(self, destination, event_id, outlier=False): """Requests the PDU with given origin and ID from the remote home server. @@ -224,7 +210,7 @@ class ReplicationLayer(object): Args: destination (str): Which home server to query pdu_origin (str): The home server that originally sent the pdu. - pdu_id (str) + event_id (str) outlier (bool): Indicates whether the PDU is an `outlier`, i.e. if it's from an arbitary point in the context as opposed to part of the current block of PDUs. Defaults to `False` @@ -233,8 +219,9 @@ class ReplicationLayer(object): Deferred: Results in the requested PDU. """ - transaction_data = yield self.transport_layer.get_pdu( - destination, pdu_origin, pdu_id) + transaction_data = yield self.transport_layer.get_event( + destination, event_id + ) transaction = Transaction(**transaction_data) @@ -249,8 +236,7 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def get_state_for_context(self, destination, context, pdu_id=None, - pdu_origin=None): + def get_state_for_context(self, destination, context, event_id=None): """Requests all of the `current` state PDUs for a given context from a remote home server. @@ -263,7 +249,9 @@ class ReplicationLayer(object): """ transaction_data = yield self.transport_layer.get_context_state( - destination, context, pdu_id=pdu_id, pdu_origin=pdu_origin, + destination, + context, + event_id=event_id, ) transaction = Transaction(**transaction_data) @@ -352,10 +340,10 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def on_context_state_request(self, context, pdu_id, pdu_origin): - if pdu_id and pdu_origin: + def on_context_state_request(self, context, event_id): + if event_id: pdus = yield self.handler.get_state_for_pdu( - pdu_id, pdu_origin + event_id ) else: raise NotImplementedError("Specify an event") @@ -370,8 +358,8 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def on_pdu_request(self, pdu_origin, pdu_id): - pdu = yield self._get_persisted_pdu(pdu_id, pdu_origin) + def on_pdu_request(self, event_id): + pdu = yield self._get_persisted_pdu(event_id) if pdu: defer.returnValue( @@ -443,9 +431,8 @@ class ReplicationLayer(object): def send_join(self, destination, pdu): _, content = yield self.transport_layer.send_join( destination, - pdu.context, - pdu.pdu_id, - pdu.origin, + pdu.room_id, + pdu.event_id, pdu.get_dict(), ) @@ -457,13 +444,13 @@ class ReplicationLayer(object): defer.returnValue(pdus) @log_function - def _get_persisted_pdu(self, pdu_id, pdu_origin): + def _get_persisted_pdu(self, event_id): """ Get a PDU from the database with given origin and id. Returns: Deferred: Results in a `Pdu`. """ - return self.handler.get_persisted_pdu(pdu_id, pdu_origin) + return self.handler.get_persisted_pdu(event_id) def _transaction_from_pdus(self, pdu_list): """Returns a new Transaction containing the given PDUs suitable for @@ -487,10 +474,10 @@ class ReplicationLayer(object): @log_function def _handle_new_pdu(self, origin, pdu, backfilled=False): # We reprocess pdus when we have seen them only as outliers - existing = yield self._get_persisted_pdu(pdu.pdu_id, pdu.origin) + existing = yield self._get_persisted_pdu(pdu.event_id) if existing and (not existing.outlier or pdu.outlier): - logger.debug("Already seen pdu %s %s", pdu.pdu_id, pdu.origin) + logger.debug("Already seen pdu %s", pdu.event_id) defer.returnValue({}) return @@ -500,23 +487,22 @@ class ReplicationLayer(object): if not pdu.outlier: # We only backfill backwards to the min depth. min_depth = yield self.handler.get_min_depth_for_context( - pdu.context + pdu.room_id ) if min_depth and pdu.depth > min_depth: - for pdu_id, origin, hashes in pdu.prev_pdus: - exists = yield self._get_persisted_pdu(pdu_id, origin) + for event_id, hashes in pdu.prev_events: + exists = yield self._get_persisted_pdu(event_id) if not exists: - logger.debug("Requesting pdu %s %s", pdu_id, origin) + logger.debug("Requesting pdu %s", event_id) try: yield self.get_pdu( pdu.origin, - pdu_id=pdu_id, - pdu_origin=origin + event_id=event_id, ) - logger.debug("Processed pdu %s %s", pdu_id, origin) + logger.debug("Processed pdu %s", event_id) except: # TODO(erikj): Do some more intelligent retries. logger.exception("Failed to get PDU") @@ -524,7 +510,7 @@ class ReplicationLayer(object): # We need to get the state at this event, since we have reached # a backward extremity edge. state = yield self.get_state_for_context( - origin, pdu.context, pdu.pdu_id, pdu.origin, + origin, pdu.room_id, pdu.event_id, ) # Persist the Pdu, but don't mark it as processed yet. diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 7f01b4faaf..04ad7e63ae 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -72,8 +72,7 @@ class TransportLayer(object): self.received_handler = None @log_function - def get_context_state(self, destination, context, pdu_id=None, - pdu_origin=None): + def get_context_state(self, destination, context, event_id=None): """ Requests all state for a given context (i.e. room) from the given server. @@ -91,60 +90,59 @@ class TransportLayer(object): subpath = "/state/%s/" % context args = {} - if pdu_id and pdu_origin: - args["pdu_id"] = pdu_id - args["pdu_origin"] = pdu_origin + if event_id: + args["event_id"] = event_id return self._do_request_for_transaction( destination, subpath, args=args ) @log_function - def get_pdu(self, destination, pdu_origin, pdu_id): + def get_event(self, destination, event_id): """ Requests the pdu with give id and origin from the given server. Args: destination (str): The host name of the remote home server we want to get the state from. - pdu_origin (str): The home server which created the PDU. - pdu_id (str): The id of the PDU being requested. + event_id (str): The id of the event being requested. Returns: Deferred: Results in a dict received from the remote homeserver. """ - logger.debug("get_pdu dest=%s, pdu_origin=%s, pdu_id=%s", - destination, pdu_origin, pdu_id) + logger.debug("get_pdu dest=%s, event_id=%s", + destination, event_id) - subpath = "/pdu/%s/%s/" % (pdu_origin, pdu_id) + subpath = "/event/%s/" % (event_id, ) return self._do_request_for_transaction(destination, subpath) @log_function - def backfill(self, dest, context, pdu_tuples, limit): + def backfill(self, dest, context, event_tuples, limit): """ Requests `limit` previous PDUs in a given context before list of PDUs. Args: dest (str) context (str) - pdu_tuples (list) + event_tuples (list) limt (int) Returns: Deferred: Results in a dict received from the remote homeserver. """ logger.debug( - "backfill dest=%s, context=%s, pdu_tuples=%s, limit=%s", - dest, context, repr(pdu_tuples), str(limit) + "backfill dest=%s, context=%s, event_tuples=%s, limit=%s", + dest, context, repr(event_tuples), str(limit) ) - if not pdu_tuples: + if not event_tuples: + # TODO: raise? return - subpath = "/backfill/%s/" % context + subpath = "/backfill/%s/" % (context,) args = { - "v": ["%s,%s" % (i, o) for i, o in pdu_tuples], + "v": event_tuples, "limit": limit, } @@ -222,11 +220,10 @@ class TransportLayer(object): @defer.inlineCallbacks @log_function - def send_join(self, destination, context, pdu_id, origin, content): - path = PREFIX + "/send_join/%s/%s/%s" % ( + def send_join(self, destination, context, event_id, content): + path = PREFIX + "/send_join/%s/%s" % ( context, - origin, - pdu_id, + event_id, ) code, content = yield self.client.put_json( @@ -242,11 +239,10 @@ class TransportLayer(object): @defer.inlineCallbacks @log_function - def send_invite(self, destination, context, pdu_id, origin, content): - path = PREFIX + "/invite/%s/%s/%s" % ( + def send_invite(self, destination, context, event_id, content): + path = PREFIX + "/invite/%s/%s" % ( context, - origin, - pdu_id, + event_id, ) code, content = yield self.client.put_json( @@ -376,10 +372,10 @@ class TransportLayer(object): # data_id pair. self.server.register_path( "GET", - re.compile("^" + PREFIX + "/pdu/([^/]*)/([^/]*)/$"), + re.compile("^" + PREFIX + "/event/([^/]*)/$"), self._with_authentication( - lambda origin, content, query, pdu_origin, pdu_id: - handler.on_pdu_request(pdu_origin, pdu_id) + lambda origin, content, query, event_id: + handler.on_pdu_request(event_id) ) ) @@ -391,8 +387,7 @@ class TransportLayer(object): lambda origin, content, query, context: handler.on_context_state_request( context, - query.get("pdu_id", [None])[0], - query.get("pdu_origin", [None])[0] + query.get("event_id", [None])[0], ) ) ) @@ -442,9 +437,9 @@ class TransportLayer(object): self.server.register_path( "PUT", - re.compile("^" + PREFIX + "/send_join/([^/]*)/([^/]*)/([^/]*)$"), + re.compile("^" + PREFIX + "/send_join/([^/]*)/([^/]*)$"), self._with_authentication( - lambda origin, content, query, context, pdu_origin, pdu_id: + lambda origin, content, query, context, event_id: self._on_send_join_request( origin, content, query, ) @@ -453,9 +448,9 @@ class TransportLayer(object): self.server.register_path( "PUT", - re.compile("^" + PREFIX + "/invite/([^/]*)/([^/]*)/([^/]*)$"), + re.compile("^" + PREFIX + "/invite/([^/]*)/([^/]*)$"), self._with_authentication( - lambda origin, content, query, context, pdu_origin, pdu_id: + lambda origin, content, query, context, event_id: self._on_invite_request( origin, content, query, ) @@ -548,7 +543,7 @@ class TransportLayer(object): limit = int(limits[-1]) - versions = [v.split(",", 1) for v in v_list] + versions = v_list return self.request_handler.on_backfill_request( context, versions, limit @@ -579,120 +574,3 @@ class TransportLayer(object): ) defer.returnValue((200, content)) - - -class TransportReceivedHandler(object): - """ Callbacks used when we receive a transaction - """ - def on_incoming_transaction(self, transaction): - """ Called on PUT /send/, or on response to a request - that we sent (e.g. a backfill request) - - Args: - transaction (synapse.transaction.Transaction): The transaction that - was sent to us. - - Returns: - twisted.internet.defer.Deferred: A deferred that gets fired when - the transaction has finished being processed. - - The result should be a tuple in the form of - `(response_code, respond_body)`, where `response_body` is a python - dict that will get serialized to JSON. - - On errors, the dict should have an `error` key with a brief message - of what went wrong. - """ - pass - - -class TransportRequestHandler(object): - """ Handlers used when someone want's data from us - """ - def on_pull_request(self, versions): - """ Called on GET /pull/?v=... - - This is hit when a remote home server wants to get all data - after a given transaction. Mainly used when a home server comes back - online and wants to get everything it has missed. - - Args: - versions (list): A list of transaction_ids that should be used to - determine what PDUs the remote side have not yet seen. - - Returns: - Deferred: Resultsin a tuple in the form of - `(response_code, respond_body)`, where `response_body` is a python - dict that will get serialized to JSON. - - On errors, the dict should have an `error` key with a brief message - of what went wrong. - """ - pass - - def on_pdu_request(self, pdu_origin, pdu_id): - """ Called on GET /pdu/// - - Someone wants a particular PDU. This PDU may or may not have originated - from us. - - Args: - pdu_origin (str) - pdu_id (str) - - Returns: - Deferred: Resultsin a tuple in the form of - `(response_code, respond_body)`, where `response_body` is a python - dict that will get serialized to JSON. - - On errors, the dict should have an `error` key with a brief message - of what went wrong. - """ - pass - - def on_context_state_request(self, context): - """ Called on GET /state// - - Gets hit when someone wants all the *current* state for a given - contexts. - - Args: - context (str): The name of the context that we're interested in. - - Returns: - twisted.internet.defer.Deferred: A deferred that gets fired when - the transaction has finished being processed. - - The result should be a tuple in the form of - `(response_code, respond_body)`, where `response_body` is a python - dict that will get serialized to JSON. - - On errors, the dict should have an `error` key with a brief message - of what went wrong. - """ - pass - - def on_backfill_request(self, context, versions, limit): - """ Called on GET /backfill//?v=...&limit=... - - Gets hit when we want to backfill backwards on a given context from - the given point. - - Args: - context (str): The context to backfill - versions (list): A list of 2-tuples representing where to backfill - from, in the form `(pdu_id, origin)` - limit (int): How many pdus to return. - - Returns: - Deferred: Results in a tuple in the form of - `(response_code, respond_body)`, where `response_body` is a python - dict that will get serialized to JSON. - - On errors, the dict should have an `error` key with a brief message - of what went wrong. - """ - pass - - def on_query_request(self): - """ Called on a GET /query/ request. """ diff --git a/synapse/federation/units.py b/synapse/federation/units.py index adc3385644..c94dcf64cf 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -34,13 +34,13 @@ class Pdu(JsonEncodedObject): A Pdu can be classified as "state". For a given context, we can efficiently retrieve all state pdu's that haven't been clobbered. Clobbering is done - via a unique constraint on the tuple (context, pdu_type, state_key). A pdu + via a unique constraint on the tuple (context, type, state_key). A pdu is a state pdu if `is_state` is True. Example pdu:: { - "pdu_id": "78c", + "event_id": "$78c:example.com", "origin_server_ts": 1404835423000, "origin": "bar", "prev_ids": [ @@ -53,14 +53,14 @@ class Pdu(JsonEncodedObject): """ valid_keys = [ - "pdu_id", - "context", + "event_id", + "room_id", "origin", "origin_server_ts", - "pdu_type", + "type", "destinations", "transaction_id", - "prev_pdus", + "prev_events", "depth", "content", "outlier", @@ -68,8 +68,7 @@ class Pdu(JsonEncodedObject): "signatures", "is_state", # Below this are keys valid only for State Pdus. "state_key", - "prev_state_id", - "prev_state_origin", + "prev_state", "required_power_level", "user_id", ] @@ -81,18 +80,18 @@ class Pdu(JsonEncodedObject): ] required_keys = [ - "pdu_id", - "context", + "event_id", + "room_id", "origin", "origin_server_ts", - "pdu_type", + "type", "content", ] # TODO: We need to make this properly load content rather than # just leaving it as a dict. (OR DO WE?!) - def __init__(self, destinations=[], is_state=False, prev_pdus=[], + def __init__(self, destinations=[], is_state=False, prev_events=[], outlier=False, hashes={}, signatures={}, **kwargs): if is_state: for required_key in ["state_key"]: @@ -102,66 +101,13 @@ class Pdu(JsonEncodedObject): super(Pdu, self).__init__( destinations=destinations, is_state=bool(is_state), - prev_pdus=prev_pdus, + prev_events=prev_events, outlier=outlier, hashes=hashes, signatures=signatures, **kwargs ) - @classmethod - def from_pdu_tuple(cls, pdu_tuple): - """ Converts a PduTuple to a Pdu - - Args: - pdu_tuple (synapse.persistence.transactions.PduTuple): The tuple to - convert - - Returns: - Pdu - """ - if pdu_tuple: - d = copy.copy(pdu_tuple.pdu_entry._asdict()) - d["origin_server_ts"] = d.pop("ts") - - for k in d.keys(): - if d[k] is None: - del d[k] - - d["content"] = json.loads(d["content_json"]) - del d["content_json"] - - args = {f: d[f] for f in cls.valid_keys if f in d} - if "unrecognized_keys" in d and d["unrecognized_keys"]: - args.update(json.loads(d["unrecognized_keys"])) - - hashes = { - alg: encode_base64(hsh) - for alg, hsh in pdu_tuple.hashes.items() - } - - signatures = { - kid: encode_base64(sig) - for kid, sig in pdu_tuple.signatures.items() - } - - prev_pdus = [] - for prev_pdu in pdu_tuple.prev_pdu_list: - prev_hashes = pdu_tuple.edge_hashes.get(prev_pdu, {}) - prev_hashes = { - alg: encode_base64(hsh) for alg, hsh in prev_hashes.items() - } - prev_pdus.append((prev_pdu[0], prev_pdu[1], prev_hashes)) - - return Pdu( - prev_pdus=prev_pdus, - hashes=hashes, - signatures=signatures, - **args - ) - else: - return None - def __str__(self): return "(%s, %s)" % (self.__class__.__name__, repr(self.__dict__)) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 18cb1d4e97..bdd28f04bb 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -139,7 +139,7 @@ class FederationHandler(BaseHandler): # Huh, let's try and get the current state try: yield self.replication_layer.get_state_for_context( - event.origin, event.room_id, pdu.pdu_id, pdu.origin, + event.origin, event.room_id, event.event_id, ) hosts = yield self.store.get_joined_hosts_for_room( @@ -368,11 +368,9 @@ class FederationHandler(BaseHandler): ]) @defer.inlineCallbacks - def get_state_for_pdu(self, pdu_id, pdu_origin): + def get_state_for_pdu(self, event_id): yield run_on_reactor() - event_id = EventID.create(pdu_id, pdu_origin, self.hs).to_string() - state_groups = yield self.store.get_state_groups( [event_id] ) @@ -406,7 +404,7 @@ class FederationHandler(BaseHandler): events = yield self.store.get_backfill_events( context, - [self.pdu_codec.encode_event_id(i, o) for i, o in pdu_list], + pdu_list, limit ) @@ -417,14 +415,14 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks @log_function - def get_persisted_pdu(self, pdu_id, origin): + def get_persisted_pdu(self, event_id): """ Get a PDU from the database with given origin and id. Returns: Deferred: Results in a `Pdu`. """ event = yield self.store.get_event( - self.pdu_codec.encode_event_id(pdu_id, origin), + event_id, allow_none=True, ) From af7ae048f8868a58dc1b82a17ca701f28624b065 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 15:06:40 +0000 Subject: [PATCH 052/227] Add option to not bind to HTTPS port. This is useful if running behind an ssl load balancer --- demo/start.sh | 2 +- synapse/app/homeserver.py | 5 ++++- synapse/config/server.py | 3 +++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/demo/start.sh b/demo/start.sh index fc6cd6303f..0530f0a26e 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -32,7 +32,7 @@ for port in 8080 8081 8082; do -D --pid-file "$DIR/$port.pid" \ --manhole $((port + 1000)) \ --tls-dh-params-path "demo/demo.tls.dh" \ - $PARAMS + $PARAMS $SYNAPSE_PARAMS python -m synapse.app.homeserver \ --config-path "demo/etc/$port.config" \ diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 6394bc27d1..a20376b9d6 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -233,7 +233,10 @@ def setup(): f.namespace['hs'] = hs reactor.listenTCP(config.manhole, f, interface='127.0.0.1') - hs.start_listening(config.bind_port, config.unsecure_port) + bind_port = config.bind_port + if config.no_tls: + bind_port = None + hs.start_listening(bind_port, config.unsecure_port) if config.daemonize: print config.pid_file diff --git a/synapse/config/server.py b/synapse/config/server.py index 3afda12d5a..814a4c349b 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -30,6 +30,7 @@ class ServerConfig(Config): self.pid_file = self.abspath(args.pid_file) self.webclient = True self.manhole = args.manhole + self.no_tls = args.no_tls if not args.content_addr: host = args.server_name @@ -67,6 +68,8 @@ class ServerConfig(Config): server_group.add_argument("--content-addr", default=None, help="The host and scheme to use for the " "content repository") + server_group.add_argument("--no-tls", action='store_true', + help="Don't bind to the https port.") def read_signing_key(self, signing_key_path): signing_keys = self.read_file(signing_key_path, "signing_key") From 2221a13a4da2a417a8b8dcc09350f884cdb10197 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 3 Nov 2014 15:58:00 +0000 Subject: [PATCH 053/227] script for checking signatures on signed json --- scripts/check_signature.py | 70 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 scripts/check_signature.py diff --git a/scripts/check_signature.py b/scripts/check_signature.py new file mode 100644 index 0000000000..6309c32cc1 --- /dev/null +++ b/scripts/check_signature.py @@ -0,0 +1,70 @@ + +from synapse.crypto.event_signing import verify_signed_event_pdu +from syutil.crypto.jsonsign import verify_signed_json +from syutil.crypto.signing_key import ( + decode_verify_key_bytes, write_signing_keys +) +from syutil.base64util import decode_base64 + +import urllib2 +import json +import sys +import dns.resolver +import pprint +import argparse +import logging + +def get_targets(server_name): + try: + answers = dns.resolver.query("_matrix._tcp." + server_name, "SRV") + for srv in answers: + yield (srv.target, srv.port) + except dns.resolver.NXDOMAIN: + yield (server_name, 8480) + +def get_server_keys(server_name, target, port): + url = "https://%s:%i/_matrix/key/v1" % (target, port) + keys = json.load(urllib2.urlopen(url)) + verify_keys = {} + for key_id, key_base64 in keys["verify_keys"].items(): + verify_key = decode_verify_key_bytes(key_id, decode_base64(key_base64)) + verify_signed_json(keys, server_name, verify_key) + verify_keys[key_id] = verify_key + return verify_keys + +def main(): + + parser = argparse.ArgumentParser() + parser.add_argument("signature_name") + parser.add_argument("input_json", nargs="?", type=argparse.FileType('r'), + default=sys.stdin) + + args = parser.parse_args() + logging.basicConfig() + + server_name = args.signature_name + keys = {} + for target, port in get_targets(server_name): + try: + keys = get_server_keys(server_name, target, port) + print "Using keys from https://%s:%s/_matrix/key/v1" % (target, port) + write_signing_keys(sys.stdout, keys.values()) + break + except: + logging.exception("Error talking to %s:%s", target, port) + + json_to_check = json.load(args.input_json) + print "Checking JSON:" + for key_id in json_to_check["signatures"][args.signature_name]: + try: + key = keys[key_id] + verify_signed_json(json_to_check, args.signature_name, key) + print "PASS %s" % (key_id,) + except: + logging.exception("Check for key %s failed" % (key_id,)) + print "FAIL %s" % (key_id,) + + +if __name__ == '__main__': + main() + From fe6832fae85d56b0b4cdc906a73a2c64a26de15d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 3 Nov 2014 16:08:22 +0000 Subject: [PATCH 054/227] handle server names with embeded ports --- scripts/check_signature.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/check_signature.py b/scripts/check_signature.py index 6309c32cc1..e7964e7e71 100644 --- a/scripts/check_signature.py +++ b/scripts/check_signature.py @@ -15,6 +15,10 @@ import argparse import logging def get_targets(server_name): + if ":" in server_name: + target, port = server_name.split(":") + yield (target, int(port)) + return try: answers = dns.resolver.query("_matrix._tcp." + server_name, "SRV") for srv in answers: From af83bf6712629bd9121d1b28b18dda936f0c28f2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 3 Nov 2014 16:35:24 +0000 Subject: [PATCH 055/227] Script for checking event hashes --- scripts/check_event_hash.py | 43 +++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 scripts/check_event_hash.py diff --git a/scripts/check_event_hash.py b/scripts/check_event_hash.py new file mode 100644 index 0000000000..9fa4452ee6 --- /dev/null +++ b/scripts/check_event_hash.py @@ -0,0 +1,43 @@ +from synapse.crypto.event_signing import * +from syutil.base64util import encode_base64 + +import argparse +import hashlib +import sys +import json + +class dictobj(dict): + def __init__(self, *args, **kargs): + dict.__init__(self, *args, **kargs) + self.__dict__ = self + + def get_dict(self): + return dict(self) + + +def main(): + parser = parser = argparse.ArgumentParser() + parser.add_argument("input_json", nargs="?", type=argparse.FileType('r'), + default=sys.stdin) + args = parser.parse_args() + logging.basicConfig() + + event_json = dictobj(json.load(args.input_json)) + + algorithms = { + "sha256": hashlib.sha256, + } + + for alg_name in event_json.hashes: + if check_event_pdu_content_hash(event_json, algorithms[alg_name]): + print "PASS content hash %s" % (alg_name,) + else: + print "FAIL content hash %s" % (alg_name,) + + for algorithm in algorithms.values(): + name, h_bytes = compute_pdu_event_reference_hash(event_json, algorithm) + print "Reference hash %s: %s" % (name, encode_base64(bytes)) + +if __name__=="__main__": + main() + From 68698e0ac8c39083f6ab7d377a48b5bead3d3598 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 17:51:42 +0000 Subject: [PATCH 056/227] Fix bugs in generating event signatures and hashing --- scripts/check_event_hash.py | 12 ++-- scripts/check_signature.py | 1 - synapse/api/events/__init__.py | 1 + synapse/crypto/event_signing.py | 116 +++++++++++++------------------- synapse/federation/pdu_codec.py | 13 +--- synapse/federation/units.py | 11 +-- 6 files changed, 58 insertions(+), 96 deletions(-) diff --git a/scripts/check_event_hash.py b/scripts/check_event_hash.py index 9fa4452ee6..7c32f8102a 100644 --- a/scripts/check_event_hash.py +++ b/scripts/check_event_hash.py @@ -6,6 +6,7 @@ import hashlib import sys import json + class dictobj(dict): def __init__(self, *args, **kargs): dict.__init__(self, *args, **kargs) @@ -14,9 +15,12 @@ class dictobj(dict): def get_dict(self): return dict(self) + def get_full_dict(self): + return dict(self) + def main(): - parser = parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser() parser.add_argument("input_json", nargs="?", type=argparse.FileType('r'), default=sys.stdin) args = parser.parse_args() @@ -29,14 +33,14 @@ def main(): } for alg_name in event_json.hashes: - if check_event_pdu_content_hash(event_json, algorithms[alg_name]): + if check_event_content_hash(event_json, algorithms[alg_name]): print "PASS content hash %s" % (alg_name,) else: print "FAIL content hash %s" % (alg_name,) for algorithm in algorithms.values(): - name, h_bytes = compute_pdu_event_reference_hash(event_json, algorithm) - print "Reference hash %s: %s" % (name, encode_base64(bytes)) + name, h_bytes = compute_event_reference_hash(event_json, algorithm) + print "Reference hash %s: %s" % (name, encode_base64(h_bytes)) if __name__=="__main__": main() diff --git a/scripts/check_signature.py b/scripts/check_signature.py index e7964e7e71..e146e18e24 100644 --- a/scripts/check_signature.py +++ b/scripts/check_signature.py @@ -1,5 +1,4 @@ -from synapse.crypto.event_signing import verify_signed_event_pdu from syutil.crypto.jsonsign import verify_signed_json from syutil.crypto.signing_key import ( decode_verify_key_bytes, write_signing_keys diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index b855811b98..168b812311 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -61,6 +61,7 @@ class SynapseEvent(JsonEncodedObject): "prev_content", "prev_state", "redacted_because", + "origin_server_ts", ] internal_keys = [ diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 0e8bc7eb6c..de5d2e7465 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -15,11 +15,11 @@ # limitations under the License. -from synapse.federation.units import Pdu -from synapse.api.events.utils import prune_pdu, prune_event +from synapse.api.events.utils import prune_event from syutil.jsonutil import encode_canonical_json from syutil.base64util import encode_base64, decode_base64 -from syutil.crypto.jsonsign import sign_json, verify_signed_json +from syutil.crypto.jsonsign import sign_json +from synapse.api.events.room import GenericEvent import copy import hashlib @@ -28,20 +28,14 @@ import logging logger = logging.getLogger(__name__) -def add_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): - hashed = _compute_content_hash(pdu, hash_algorithm) - pdu.hashes[hashed.name] = encode_base64(hashed.digest()) - return pdu - - -def check_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): +def check_event_content_hash(event, hash_algorithm=hashlib.sha256): """Check whether the hash for this PDU matches the contents""" - computed_hash = _compute_content_hash(pdu, hash_algorithm) - if computed_hash.name not in pdu.hashes: + computed_hash = _compute_content_hash(event, hash_algorithm) + if computed_hash.name not in event.hashes: raise Exception("Algorithm %s not in hashes %s" % ( - computed_hash.name, list(pdu.hashes) + computed_hash.name, list(event.hashes) )) - message_hash_base64 = pdu.hashes[computed_hash.name] + message_hash_base64 = event.hashes[computed_hash.name] try: message_hash_bytes = decode_base64(message_hash_base64) except: @@ -49,62 +43,7 @@ def check_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): return message_hash_bytes == computed_hash.digest() -def _compute_content_hash(pdu, hash_algorithm): - pdu_json = pdu.get_dict() - #TODO: Make "age_ts" key internal - pdu_json.pop("age_ts", None) - pdu_json.pop("unsigned", None) - pdu_json.pop("signatures", None) - pdu_json.pop("hashes", None) - pdu_json_bytes = encode_canonical_json(pdu_json) - return hash_algorithm(pdu_json_bytes) - - -def compute_pdu_event_reference_hash(pdu, hash_algorithm=hashlib.sha256): - tmp_pdu = Pdu(**pdu.get_dict()) - tmp_pdu = prune_pdu(tmp_pdu) - pdu_json = tmp_pdu.get_dict() - pdu_json.pop("signatures", None) - pdu_json_bytes = encode_canonical_json(pdu_json) - hashed = hash_algorithm(pdu_json_bytes) - return (hashed.name, hashed.digest()) - - -def compute_event_reference_hash(event, hash_algorithm=hashlib.sha256): - tmp_event = copy.deepcopy(event) - tmp_event = prune_event(tmp_event) - event_json = tmp_event.get_dict() - event_json.pop("signatures", None) - event_json_bytes = encode_canonical_json(event_json) - hashed = hash_algorithm(event_json_bytes) - return (hashed.name, hashed.digest()) - - -def sign_event_pdu(pdu, signature_name, signing_key): - tmp_pdu = Pdu(**pdu.get_dict()) - tmp_pdu = prune_pdu(tmp_pdu) - pdu_json = tmp_pdu.get_dict() - pdu_json = sign_json(pdu_json, signature_name, signing_key) - pdu.signatures = pdu_json["signatures"] - return pdu - - -def verify_signed_event_pdu(pdu, signature_name, verify_key): - tmp_pdu = Pdu(**pdu.get_dict()) - tmp_pdu = prune_pdu(tmp_pdu) - pdu_json = tmp_pdu.get_dict() - verify_signed_json(pdu_json, signature_name, verify_key) - - -def add_hashes_and_signatures(event, signature_name, signing_key, - hash_algorithm=hashlib.sha256): - tmp_event = copy.deepcopy(event) - tmp_event = prune_event(tmp_event) - redact_json = tmp_event.get_dict() - redact_json.pop("signatures", None) - redact_json = sign_json(redact_json, signature_name, signing_key) - event.signatures = redact_json["signatures"] - +def _compute_content_hash(event, hash_algorithm): event_json = event.get_full_dict() #TODO: We need to sign the JSON that is going out via fedaration. event_json.pop("age_ts", None) @@ -112,7 +51,44 @@ def add_hashes_and_signatures(event, signature_name, signing_key, event_json.pop("signatures", None) event_json.pop("hashes", None) event_json_bytes = encode_canonical_json(event_json) + return hash_algorithm(event_json_bytes) + + +def compute_event_reference_hash(event, hash_algorithm=hashlib.sha256): + # FIXME(erikj): GenericEvent! + tmp_event = GenericEvent(**event.get_full_dict()) + tmp_event = prune_event(tmp_event) + event_json = tmp_event.get_dict() + event_json.pop("signatures", None) + event_json.pop("age_ts", None) + event_json.pop("unsigned", None) + event_json_bytes = encode_canonical_json(event_json) hashed = hash_algorithm(event_json_bytes) + return (hashed.name, hashed.digest()) + + +def compute_event_signature(event, signature_name, signing_key): + tmp_event = copy.deepcopy(event) + tmp_event = prune_event(tmp_event) + redact_json = tmp_event.get_full_dict() + redact_json.pop("signatures", None) + redact_json.pop("age_ts", None) + redact_json.pop("unsigned", None) + logger.debug("Signing event: %s", redact_json) + redact_json = sign_json(redact_json, signature_name, signing_key) + return redact_json["signatures"] + + +def add_hashes_and_signatures(event, signature_name, signing_key, + hash_algorithm=hashlib.sha256): + hashed = _compute_content_hash(event, hash_algorithm=hash_algorithm) + if not hasattr(event, "hashes"): event.hashes = {} event.hashes[hashed.name] = encode_base64(hashed.digest()) + + event.signatures = compute_event_signature( + event, + signature_name=signature_name, + signing_key=signing_key, + ) diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index 5ec97a698e..52c84efb5b 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -14,10 +14,6 @@ # limitations under the License. from .units import Pdu -from synapse.crypto.event_signing import ( - add_event_pdu_content_hash, sign_event_pdu -) -from synapse.types import EventID import copy @@ -49,17 +45,10 @@ class PduCodec(object): def pdu_from_event(self, event): d = event.get_full_dict() - if hasattr(event, "state_key"): - d["is_state"] = True - kwargs = copy.deepcopy(event.unrecognized_keys) kwargs.update({ k: v for k, v in d.items() }) - if "origin_server_ts" not in kwargs: - kwargs["origin_server_ts"] = int(self.clock.time_msec()) - pdu = Pdu(**kwargs) - pdu = add_event_pdu_content_hash(pdu) - return sign_event_pdu(pdu, self.server_name, self.signing_key) + return pdu diff --git a/synapse/federation/units.py b/synapse/federation/units.py index c94dcf64cf..c2d8dca8f3 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -65,8 +65,7 @@ class Pdu(JsonEncodedObject): "content", "outlier", "hashes", - "signatures", - "is_state", # Below this are keys valid only for State Pdus. + "signatures", # Below this are keys valid only for State Pdus. "state_key", "prev_state", "required_power_level", @@ -91,16 +90,10 @@ class Pdu(JsonEncodedObject): # TODO: We need to make this properly load content rather than # just leaving it as a dict. (OR DO WE?!) - def __init__(self, destinations=[], is_state=False, prev_events=[], + def __init__(self, destinations=[], prev_events=[], outlier=False, hashes={}, signatures={}, **kwargs): - if is_state: - for required_key in ["state_key"]: - if required_key not in kwargs: - raise RuntimeError("Key %s is required" % required_key) - super(Pdu, self).__init__( destinations=destinations, - is_state=bool(is_state), prev_events=prev_events, outlier=outlier, hashes=hashes, From aa76bf39aba2eadf506f57952a1dffce629f2637 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 14:14:02 +0000 Subject: [PATCH 057/227] Remove unused imports --- synapse/federation/persistence.py | 2 -- synapse/federation/units.py | 3 --- synapse/handlers/federation.py | 6 ++---- synapse/state.py | 2 -- synapse/storage/_base.py | 2 -- 5 files changed, 2 insertions(+), 13 deletions(-) diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index b04fbb4177..73dc844d59 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -21,8 +21,6 @@ These actions are mostly only used by the :py:mod:`.replication` module. from twisted.internet import defer -from .units import Pdu - from synapse.util.logutils import log_function import json diff --git a/synapse/federation/units.py b/synapse/federation/units.py index c2d8dca8f3..9b25556707 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -18,11 +18,8 @@ server protocol. """ from synapse.util.jsonobject import JsonEncodedObject -from syutil.base64util import encode_base64 import logging -import json -import copy logger = logging.getLogger(__name__) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index bdd28f04bb..49bfff88a4 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -17,15 +17,13 @@ from ._base import BaseHandler -from synapse.api.events.room import InviteJoinEvent, RoomMemberEvent +from synapse.api.events.room import RoomMemberEvent from synapse.api.constants import Membership from synapse.util.logutils import log_function from synapse.federation.pdu_codec import PduCodec -from synapse.api.errors import SynapseError from synapse.util.async import run_on_reactor -from synapse.types import EventID -from twisted.internet import defer, reactor +from twisted.internet import defer import logging diff --git a/synapse/state.py b/synapse/state.py index f4efc287c9..9771883bc3 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -19,8 +19,6 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor -from synapse.types import EventID - from collections import namedtuple import copy diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 464b12f032..5e00c23fd1 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -14,8 +14,6 @@ # limitations under the License. import logging -from twisted.internet import defer - from synapse.api.errors import StoreError from synapse.api.events.utils import prune_event from synapse.util.logutils import log_function From d7412c4df1cdfc1128b6d63a38c4ac1d6f08b76b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 14:16:19 +0000 Subject: [PATCH 058/227] Remove unused interface --- synapse/federation/replication.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 838e660a46..99dd390a64 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -533,14 +533,6 @@ class ReplicationLayer(object): return "" % self.server_name -class ReplicationHandler(object): - """This defines the methods that the :py:class:`.ReplicationLayer` will - use to communicate with the rest of the home server. - """ - def on_receive_pdu(self, pdu): - raise NotImplementedError("on_receive_pdu") - - class _TransactionQueue(object): """This class makes sure we only have one transaction in flight at a time for a given destination. From 440cbd5235e7e23dfe97d8e3d394cc0d35b35fd6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 14:17:55 +0000 Subject: [PATCH 059/227] Add support for sending failures --- synapse/federation/replication.py | 30 +++++++++++++++++++++++++-- synapse/federation/units.py | 1 + synapse/types.py | 34 +++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 99dd390a64..680e7322a6 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -143,6 +143,11 @@ class ReplicationLayer(object): self._transaction_queue.enqueue_edu(edu) return defer.succeed(None) + @log_function + def send_failure(self, failure, destination): + self._transaction_queue.enqueue_failure(failure, destination) + return defer.succeed(None) + @log_function def make_query(self, destination, query_type, args, retry_on_dns_fail=True): @@ -558,6 +563,9 @@ class _TransactionQueue(object): # destination -> list of tuple(edu, deferred) self.pending_edus_by_dest = {} + # destination -> list of tuple(failure, deferred) + self.pending_failures_by_dest = {} + # HACK to get unique tx id self._next_txn_id = int(self._clock.time_msec()) @@ -610,6 +618,18 @@ class _TransactionQueue(object): return deferred + @defer.inlineCallbacks + def enqueue_failure(self, failure, destination): + deferred = defer.Deferred() + + self.pending_failures_by_dest.setdefault( + destination, [] + ).append( + (failure, deferred) + ) + + yield deferred + @defer.inlineCallbacks @log_function def _attempt_new_transaction(self, destination): @@ -619,8 +639,9 @@ class _TransactionQueue(object): # list of (pending_pdu, deferred, order) pending_pdus = self.pending_pdus_by_dest.pop(destination, []) pending_edus = self.pending_edus_by_dest.pop(destination, []) + pending_failures = self.pending_failures_by_dest(destination, []) - if not pending_pdus and not pending_edus: + if not pending_pdus and not pending_edus and not pending_failures: return logger.debug("TX [%s] Attempting new transaction", destination) @@ -630,7 +651,11 @@ class _TransactionQueue(object): pdus = [x[0] for x in pending_pdus] edus = [x[0] for x in pending_edus] - deferreds = [x[1] for x in pending_pdus + pending_edus] + failures = [x[0].get_dict() for x in pending_failures] + deferreds = [ + x[1] + for x in pending_pdus + pending_edus + pending_failures + ] try: self.pending_transactions[destination] = 1 @@ -644,6 +669,7 @@ class _TransactionQueue(object): destination=destination, pdus=pdus, edus=edus, + pdu_failures=failures, ) self._next_txn_id += 1 diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 9b25556707..2070ffe1e2 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -157,6 +157,7 @@ class Transaction(JsonEncodedObject): "edus", "transaction_id", "destination", + "pdu_failures", ] internal_keys = [ diff --git a/synapse/types.py b/synapse/types.py index 649ff2f7d7..8fac20fd2e 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -128,3 +128,37 @@ class StreamToken( d = self._asdict() d[key] = new_value return StreamToken(**d) + + +class FederationError(RuntimeError): + """ This class is used to inform remote home servers about erroneous + PDUs they sent us. + + FATAL: The remote server could not interpret the source event. + (e.g., it was missing a required field) + ERROR: The remote server interpreted the event, but it failed some other + check (e.g. auth) + WARN: The remote server accepted the event, but believes some part of it + is wrong (e.g., it referred to an invalid event) + """ + + def __init__(self, level, code, reason, affected, source=None): + if level not in ["FATAL", "ERROR", "WARN"]: + raise ValueError("Level is not valid: %s" % (level,)) + self.level = level + self.code = code + self.reason = reason + self.affected = affected + self.source = source + + msg = "%s %s: %s" % (level, code, reason,) + super(FederationError, self).__init__(msg) + + def get_dict(self): + return { + "level": self.level, + "code": self.code, + "reason": self.reason, + "affected": self.affected, + "source": self.source if self.source else self.affected, + } From fc7b2b11a28b49a4d5b50fef66bae0faee47503f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 15:09:34 +0000 Subject: [PATCH 060/227] PEP8 --- synapse/federation/replication.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 680e7322a6..98997998ff 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -81,7 +81,7 @@ class ReplicationLayer(object): def register_edu_handler(self, edu_type, handler): if edu_type in self.edu_handlers: - raise KeyError("Already have an EDU handler for %s" % (edu_type)) + raise KeyError("Already have an EDU handler for %s" % (edu_type,)) self.edu_handlers[edu_type] = handler @@ -102,7 +102,9 @@ class ReplicationLayer(object): object to encode as JSON. """ if query_type in self.query_handlers: - raise KeyError("Already have a Query handler for %s" % (query_type)) + raise KeyError( + "Already have a Query handler for %s" % (query_type,) + ) self.query_handlers[query_type] = handler @@ -128,7 +130,10 @@ class ReplicationLayer(object): # TODO, add errback, etc. self._transaction_queue.enqueue_pdu(pdu, order) - logger.debug("[%s] transaction_layer.enqueue_pdu... done", pdu.event_id) + logger.debug( + "[%s] transaction_layer.enqueue_pdu... done", + pdu.event_id + ) @log_function def send_edu(self, destination, edu_type, content): @@ -317,7 +322,11 @@ class ReplicationLayer(object): if hasattr(transaction, "edus"): for edu in [Edu(**x) for x in transaction.edus]: - self.received_edu(transaction.origin, edu.edu_type, edu.content) + self.received_edu( + transaction.origin, + edu.edu_type, + edu.content + ) results = yield defer.DeferredList(dl) From 8918422156a3555a61b7e5ba2ee0c04eb3e52cb5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 15:10:27 +0000 Subject: [PATCH 061/227] Move FederationError to synapse.api.errors --- synapse/api/errors.py | 34 ++++++++++++++++++++++++++++++++++ synapse/types.py | 34 ---------------------------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 38ccb4f9d1..33d15072af 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -158,3 +158,37 @@ def cs_error(msg, code=Codes.UNKNOWN, **kwargs): for key, value in kwargs.iteritems(): err[key] = value return err + + +class FederationError(RuntimeError): + """ This class is used to inform remote home servers about erroneous + PDUs they sent us. + + FATAL: The remote server could not interpret the source event. + (e.g., it was missing a required field) + ERROR: The remote server interpreted the event, but it failed some other + check (e.g. auth) + WARN: The remote server accepted the event, but believes some part of it + is wrong (e.g., it referred to an invalid event) + """ + + def __init__(self, level, code, reason, affected, source=None): + if level not in ["FATAL", "ERROR", "WARN"]: + raise ValueError("Level is not valid: %s" % (level,)) + self.level = level + self.code = code + self.reason = reason + self.affected = affected + self.source = source + + msg = "%s %s: %s" % (level, code, reason,) + super(FederationError, self).__init__(msg) + + def get_dict(self): + return { + "level": self.level, + "code": self.code, + "reason": self.reason, + "affected": self.affected, + "source": self.source if self.source else self.affected, + } diff --git a/synapse/types.py b/synapse/types.py index 8fac20fd2e..649ff2f7d7 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -128,37 +128,3 @@ class StreamToken( d = self._asdict() d[key] = new_value return StreamToken(**d) - - -class FederationError(RuntimeError): - """ This class is used to inform remote home servers about erroneous - PDUs they sent us. - - FATAL: The remote server could not interpret the source event. - (e.g., it was missing a required field) - ERROR: The remote server interpreted the event, but it failed some other - check (e.g. auth) - WARN: The remote server accepted the event, but believes some part of it - is wrong (e.g., it referred to an invalid event) - """ - - def __init__(self, level, code, reason, affected, source=None): - if level not in ["FATAL", "ERROR", "WARN"]: - raise ValueError("Level is not valid: %s" % (level,)) - self.level = level - self.code = code - self.reason = reason - self.affected = affected - self.source = source - - msg = "%s %s: %s" % (level, code, reason,) - super(FederationError, self).__init__(msg) - - def get_dict(self): - return { - "level": self.level, - "code": self.code, - "reason": self.reason, - "affected": self.affected, - "source": self.source if self.source else self.affected, - } From 2a49f177fe39ee15c7fc2915db9aadb6152bc547 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 15:10:43 +0000 Subject: [PATCH 062/227] On AuthError, raise a FederationError --- synapse/handlers/federation.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 49bfff88a4..8848656b1d 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -17,6 +17,7 @@ from ._base import BaseHandler +from synapse.api.errors import AuthError, FederationError from synapse.api.events.room import RoomMemberEvent from synapse.api.constants import Membership from synapse.util.logutils import log_function @@ -116,8 +117,15 @@ class FederationHandler(BaseHandler): logger.debug("Event: %s", event) - if not backfilled: + try: yield self.auth.check(event, None, raises=True) + except AuthError as e: + raise FederationError( + "ERROR", + e.code, + e.msg, + affected=event.event_id, + ) is_new_state = is_new_state and not backfilled From a5a4ef3fd7b632d838940fb3bd7b079a897fa8a4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 15:16:43 +0000 Subject: [PATCH 063/227] Fix bug in replication --- synapse/federation/replication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 98997998ff..92a9678e2c 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -648,7 +648,7 @@ class _TransactionQueue(object): # list of (pending_pdu, deferred, order) pending_pdus = self.pending_pdus_by_dest.pop(destination, []) pending_edus = self.pending_edus_by_dest.pop(destination, []) - pending_failures = self.pending_failures_by_dest(destination, []) + pending_failures = self.pending_failures_by_dest.pop(destination, []) if not pending_pdus and not pending_edus and not pending_failures: return From 24305ba5bf39538f692e108a4f9dcc13c0e4ab85 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 16:15:30 +0000 Subject: [PATCH 064/227] Fix up federation tests --- tests/federation/test_federation.py | 238 +++++++++++++++------------- 1 file changed, 124 insertions(+), 114 deletions(-) diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 3a14c7c3b9..eb329eec50 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -24,7 +24,6 @@ from ..utils import MockHttpResource, MockClock, MockKey from synapse.server import HomeServer from synapse.federation import initialize_http_replication from synapse.federation.units import Pdu -from synapse.storage.pdu import PduTuple, PduEntry def make_pdu(prev_pdus=[], **kwargs): @@ -41,7 +40,7 @@ def make_pdu(prev_pdus=[], **kwargs): } pdu_fields.update(kwargs) - return PduTuple(PduEntry(**pdu_fields), prev_pdus, {}, {}, {}) + return Pdu(prev_pdus=prev_pdus, **pdu_fields) class FederationTestCase(unittest.TestCase): @@ -52,179 +51,185 @@ class FederationTestCase(unittest.TestCase): "put_json", ]) self.mock_persistence = Mock(spec=[ - "get_current_state_for_context", - "get_pdu", - "persist_event", - "update_min_depth_for_context", "prep_send_transaction", "delivered_txn", "get_received_txn_response", "set_received_txn_response", ]) self.mock_persistence.get_received_txn_response.return_value = ( - defer.succeed(None) + defer.succeed(None) ) self.mock_config = Mock() self.mock_config.signing_key = [MockKey()] self.clock = MockClock() - hs = HomeServer("test", - resource_for_federation=self.mock_resource, - http_client=self.mock_http_client, - db_pool=None, - datastore=self.mock_persistence, - clock=self.clock, - config=self.mock_config, - keyring=Mock(), + hs = HomeServer( + "test", + resource_for_federation=self.mock_resource, + http_client=self.mock_http_client, + db_pool=None, + datastore=self.mock_persistence, + clock=self.clock, + config=self.mock_config, + keyring=Mock(), ) self.federation = initialize_http_replication(hs) self.distributor = hs.get_distributor() @defer.inlineCallbacks def test_get_state(self): - self.mock_persistence.get_current_state_for_context.return_value = ( - defer.succeed([]) - ) + mock_handler = Mock(spec=[ + "get_state_for_pdu", + ]) + + self.federation.set_handler(mock_handler) + + mock_handler.get_state_for_pdu.return_value = defer.succeed([]) # Empty context initially - (code, response) = yield self.mock_resource.trigger("GET", - "/_matrix/federation/v1/state/my-context/", None) + (code, response) = yield self.mock_resource.trigger( + "GET", + "/_matrix/federation/v1/state/my-context/", + None + ) self.assertEquals(200, code) self.assertFalse(response["pdus"]) # Now lets give the context some state - self.mock_persistence.get_current_state_for_context.return_value = ( + mock_handler.get_state_for_pdu.return_value = ( defer.succeed([ make_pdu( - pdu_id="the-pdu-id", + event_id="the-pdu-id", origin="red", - context="my-context", - pdu_type="m.topic", - ts=123456789000, + room_id="my-context", + type="m.topic", + origin_server_ts=123456789000, depth=1, - is_state=True, - content_json='{"topic":"The topic"}', + content={"topic": "The topic"}, state_key="", power_level=1000, - prev_state_id="last-pdu-id", - prev_state_origin="blue", + prev_state="last-pdu-id", ), ]) ) - (code, response) = yield self.mock_resource.trigger("GET", - "/_matrix/federation/v1/state/my-context/", None) + (code, response) = yield self.mock_resource.trigger( + "GET", + "/_matrix/federation/v1/state/my-context/", + None + ) self.assertEquals(200, code) self.assertEquals(1, len(response["pdus"])) @defer.inlineCallbacks def test_get_pdu(self): - self.mock_persistence.get_pdu.return_value = ( + mock_handler = Mock(spec=[ + "get_persisted_pdu", + ]) + + self.federation.set_handler(mock_handler) + + mock_handler.get_persisted_pdu.return_value = ( defer.succeed(None) ) - (code, response) = yield self.mock_resource.trigger("GET", - "/_matrix/federation/v1/pdu/red/abc123def456/", None) + (code, response) = yield self.mock_resource.trigger( + "GET", + "/_matrix/federation/v1/event/abc123def456/", + None + ) self.assertEquals(404, code) # Now insert such a PDU - self.mock_persistence.get_pdu.return_value = ( + mock_handler.get_persisted_pdu.return_value = ( defer.succeed( make_pdu( - pdu_id="abc123def456", + event_id="abc123def456", origin="red", - context="my-context", - pdu_type="m.text", - ts=123456789001, + room_id="my-context", + type="m.text", + origin_server_ts=123456789001, depth=1, - content_json='{"text":"Here is the message"}', + content={"text": "Here is the message"}, ) ) ) - (code, response) = yield self.mock_resource.trigger("GET", - "/_matrix/federation/v1/pdu/red/abc123def456/", None) + (code, response) = yield self.mock_resource.trigger( + "GET", + "/_matrix/federation/v1/event/abc123def456/", + None + ) self.assertEquals(200, code) self.assertEquals(1, len(response["pdus"])) - self.assertEquals("m.text", response["pdus"][0]["pdu_type"]) + self.assertEquals("m.text", response["pdus"][0]["type"]) @defer.inlineCallbacks def test_send_pdu(self): self.mock_http_client.put_json.return_value = defer.succeed( - (200, "OK") + (200, "OK") ) pdu = Pdu( - pdu_id="abc123def456", - origin="red", - destinations=["remote"], - context="my-context", - origin_server_ts=123456789002, - pdu_type="m.test", - content={"testing": "content here"}, - depth=1, + event_id="abc123def456", + origin="red", + room_id="my-context", + type="m.text", + origin_server_ts=123456789001, + depth=1, + content={"text": "Here is the message"}, + destinations=["remote"], ) yield self.federation.send_pdu(pdu) self.mock_http_client.put_json.assert_called_with( - "remote", - path="/_matrix/federation/v1/send/1000000/", - data={ - "origin_server_ts": 1000000, - "origin": "test", - "pdus": [ - { - "origin": "red", - "pdu_id": "abc123def456", - "prev_pdus": [], - "origin_server_ts": 123456789002, - "context": "my-context", - "pdu_type": "m.test", - "is_state": False, - "content": {"testing": "content here"}, - "depth": 1, - "hashes": {}, - "signatures": {}, - }, - ] - }, - json_data_callback=ANY, + "remote", + path="/_matrix/federation/v1/send/1000000/", + data={ + "origin_server_ts": 1000000, + "origin": "test", + "pdus": [ + pdu.get_dict(), + ], + 'pdu_failures': [], + }, + json_data_callback=ANY, ) @defer.inlineCallbacks def test_send_edu(self): self.mock_http_client.put_json.return_value = defer.succeed( - (200, "OK") + (200, "OK") ) yield self.federation.send_edu( - destination="remote", - edu_type="m.test", - content={"testing": "content here"}, + destination="remote", + edu_type="m.test", + content={"testing": "content here"}, ) # MockClock ensures we can guess these timestamps self.mock_http_client.put_json.assert_called_with( - "remote", - path="/_matrix/federation/v1/send/1000000/", - data={ - "origin": "test", - "origin_server_ts": 1000000, - "pdus": [], - "edus": [ - { - # TODO: SYN-103: Remove "origin" and "destination" - "origin": "test", - "destination": "remote", - "edu_type": "m.test", - "content": {"testing": "content here"}, - } - ], - }, - json_data_callback=ANY, + "remote", + path="/_matrix/federation/v1/send/1000000/", + data={ + "origin": "test", + "origin_server_ts": 1000000, + "pdus": [], + "edus": [ + { + # TODO: SYN-103: Remove "origin" and "destination" + "origin": "test", + "destination": "remote", + "edu_type": "m.test", + "content": {"testing": "content here"}, + } + ], + 'pdu_failures': [], + }, + json_data_callback=ANY, ) - @defer.inlineCallbacks def test_recv_edu(self): recv_observer = Mock() @@ -232,24 +237,26 @@ class FederationTestCase(unittest.TestCase): self.federation.register_edu_handler("m.test", recv_observer) - yield self.mock_resource.trigger("PUT", - "/_matrix/federation/v1/send/1001000/", - """{ - "origin": "remote", - "origin_server_ts": 1001000, - "pdus": [], - "edus": [ - { - "origin": "remote", - "destination": "test", - "edu_type": "m.test", - "content": {"testing": "reply here"} - } - ] - }""") + yield self.mock_resource.trigger( + "PUT", + "/_matrix/federation/v1/send/1001000/", + """{ + "origin": "remote", + "origin_server_ts": 1001000, + "pdus": [], + "edus": [ + { + "origin": "remote", + "destination": "test", + "edu_type": "m.test", + "content": {"testing": "reply here"} + } + ] + }""" + ) recv_observer.assert_called_with( - "remote", {"testing": "reply here"} + "remote", {"testing": "reply here"} ) @defer.inlineCallbacks @@ -280,8 +287,11 @@ class FederationTestCase(unittest.TestCase): self.federation.register_query_handler("a-question", recv_handler) - code, response = yield self.mock_resource.trigger("GET", - "/_matrix/federation/v1/query/a-question?three=3&four=4", None) + code, response = yield self.mock_resource.trigger( + "GET", + "/_matrix/federation/v1/query/a-question?three=3&four=4", + None + ) self.assertEquals(200, code) self.assertEquals({"another": "response"}, response) From b15e8d5bbc58ccce49b852a07c5f6364e18c39cd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 16:20:02 +0000 Subject: [PATCH 065/227] event <-> pdu mappings are now trivial and will soon be scrapped --- tests/federation/test_pdu_codec.py | 167 ----------------------------- 1 file changed, 167 deletions(-) delete mode 100644 tests/federation/test_pdu_codec.py diff --git a/tests/federation/test_pdu_codec.py b/tests/federation/test_pdu_codec.py deleted file mode 100644 index fd0f72c734..0000000000 --- a/tests/federation/test_pdu_codec.py +++ /dev/null @@ -1,167 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from tests import unittest - -from synapse.federation.pdu_codec import ( - PduCodec, encode_event_id, decode_event_id -) -from synapse.federation.units import Pdu -#from synapse.api.events.room import MessageEvent - -from synapse.server import HomeServer - -from mock import Mock, NonCallableMock - -from ..utils import MockKey - - -class PduCodecTestCase(unittest.TestCase): - def setUp(self): - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - self.hs = HomeServer( - "blargle.net", - config=self.mock_config, - ) - self.event_factory = self.hs.get_event_factory() - self.codec = PduCodec(self.hs) - - def test_decode_event_id(self): - self.assertEquals( - ("foo", "bar.com"), - decode_event_id("foo@bar.com", "A") - ) - - self.assertEquals( - ("foo", "bar.com"), - decode_event_id("foo", "bar.com") - ) - - def test_encode_event_id(self): - self.assertEquals("A@B", encode_event_id("A", "B")) - - def test_codec_event_id(self): - event_id = "aa@bb.com" - - self.assertEquals( - event_id, - encode_event_id(*decode_event_id(event_id, None)) - ) - - pdu_id = ("aa", "bb.com") - - self.assertEquals( - pdu_id, - decode_event_id(encode_event_id(*pdu_id), None) - ) - - def test_event_from_pdu(self): - pdu = Pdu( - pdu_id="foo", - context="rooooom", - pdu_type="m.room.message", - origin="bar.com", - origin_server_ts=12345, - depth=5, - prev_pdus=[("alice", "bob.com")], - is_state=False, - content={"msgtype": u"test"}, - ) - - event = self.codec.event_from_pdu(pdu) - - self.assertEquals("foo@bar.com", event.event_id) - self.assertEquals(pdu.context, event.room_id) - self.assertEquals(pdu.is_state, event.is_state) - self.assertEquals(pdu.depth, event.depth) - self.assertEquals(pdu.prev_pdus, event.prev_pdus) - self.assertEquals(pdu.content, event.content) - - def test_pdu_from_event(self): - event = self.event_factory.create_event( - etype="m.room.message", - event_id="gargh_id", - room_id="rooom", - user_id="sender", - content={"msgtype": u"test"}, - ) - - pdu = self.codec.pdu_from_event(event) - - self.assertEquals(event.event_id, pdu.pdu_id) - self.assertEquals(self.hs.hostname, pdu.origin) - self.assertEquals(event.room_id, pdu.context) - self.assertEquals(event.content, pdu.content) - self.assertEquals(event.type, pdu.pdu_type) - - event = self.event_factory.create_event( - etype="m.room.message", - event_id="gargh_id@bob.com", - room_id="rooom", - user_id="sender", - content={"msgtype": u"test"}, - ) - - pdu = self.codec.pdu_from_event(event) - - self.assertEquals("gargh_id", pdu.pdu_id) - self.assertEquals("bob.com", pdu.origin) - self.assertEquals(event.room_id, pdu.context) - self.assertEquals(event.content, pdu.content) - self.assertEquals(event.type, pdu.pdu_type) - - def test_event_from_state_pdu(self): - pdu = Pdu( - pdu_id="foo", - context="rooooom", - pdu_type="m.room.topic", - origin="bar.com", - origin_server_ts=12345, - depth=5, - prev_pdus=[("alice", "bob.com")], - is_state=True, - content={"topic": u"test"}, - state_key="", - ) - - event = self.codec.event_from_pdu(pdu) - - self.assertEquals("foo@bar.com", event.event_id) - self.assertEquals(pdu.context, event.room_id) - self.assertEquals(pdu.is_state, event.is_state) - self.assertEquals(pdu.depth, event.depth) - self.assertEquals(pdu.prev_pdus, event.prev_pdus) - self.assertEquals(pdu.content, event.content) - self.assertEquals(pdu.state_key, event.state_key) - - def test_pdu_from_state_event(self): - event = self.event_factory.create_event( - etype="m.room.topic", - event_id="gargh_id", - room_id="rooom", - user_id="sender", - content={"topic": u"test"}, - ) - - pdu = self.codec.pdu_from_event(event) - - self.assertEquals(event.event_id, pdu.pdu_id) - self.assertEquals(self.hs.hostname, pdu.origin) - self.assertEquals(event.room_id, pdu.context) - self.assertEquals(event.content, pdu.content) - self.assertEquals(event.type, pdu.pdu_type) - self.assertEquals(event.state_key, pdu.state_key) From dd3711bdbd2a439208a9e22b7b15f0c3676e611f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 16:33:52 +0000 Subject: [PATCH 066/227] Fix tests.handlers.test_directory --- tests/handlers/test_directory.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index e10a49a8ac..16c446498d 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -21,9 +21,8 @@ from mock import Mock from synapse.server import HomeServer from synapse.handlers.directory import DirectoryHandler -from synapse.storage.directory import RoomAliasMapping -from tests.utils import SQLiteMemoryDbPool +from tests.utils import SQLiteMemoryDbPool, MockKey class DirectoryHandlers(object): @@ -48,11 +47,15 @@ class DirectoryTestCase(unittest.TestCase): db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() + self.mock_config = Mock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", db_pool=db_pool, http_client=None, resource_for_federation=Mock(), replication_layer=self.mock_federation, + config=self.mock_config, ) hs.handlers = DirectoryHandlers(hs) From 7f4c7fe4e823eeacaa5b8c83201d504de53beefe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 16:35:38 +0000 Subject: [PATCH 067/227] PEP8 --- tests/handlers/test_directory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index 16c446498d..8e164e4be0 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -40,6 +40,7 @@ class DirectoryTestCase(unittest.TestCase): ]) self.query_handlers = {} + def register_query_handler(query_type, handler): self.query_handlers[query_type] = handler self.mock_federation.register_query_handler = register_query_handler @@ -50,7 +51,8 @@ class DirectoryTestCase(unittest.TestCase): self.mock_config = Mock() self.mock_config.signing_key = [MockKey()] - hs = HomeServer("test", + hs = HomeServer( + "test", db_pool=db_pool, http_client=None, resource_for_federation=Mock(), From da4a09f977181ae222438dc75f5717cbebc6fe86 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 16:51:23 +0000 Subject: [PATCH 068/227] Don't bother locking --- synapse/handlers/federation.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 8848656b1d..06a2dabae2 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -132,12 +132,11 @@ class FederationHandler(BaseHandler): # TODO: Implement something in federation that allows us to # respond to PDU. - with (yield self.room_lock.lock(event.room_id)): - yield self.store.persist_event( - event, - backfilled, - is_new_state=is_new_state - ) + yield self.store.persist_event( + event, + backfilled, + is_new_state=is_new_state + ) room = yield self.store.get_room(event.room_id) From b0554682ed0c037f8726f3e37dba2113b5e8764f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 16:51:59 +0000 Subject: [PATCH 069/227] Fix federation handler tests. I've removed the invite/join dance ones as they are completely out of date. --- tests/handlers/test_federation.py | 84 ++++++++++--------------------- 1 file changed, 27 insertions(+), 57 deletions(-) diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 219b2c4c5e..63a9990f7a 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -17,16 +17,15 @@ from twisted.internet import defer from tests import unittest from synapse.api.events.room import ( - InviteJoinEvent, MessageEvent, RoomMemberEvent + MessageEvent, ) -from synapse.api.constants import Membership from synapse.handlers.federation import FederationHandler from synapse.server import HomeServer from synapse.federation.units import Pdu from mock import NonCallableMock, ANY -from ..utils import get_mock_call_args, MockKey +from ..utils import MockKey class FederationTestCase(unittest.TestCase): @@ -36,6 +35,14 @@ class FederationTestCase(unittest.TestCase): self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] + self.state_handler = NonCallableMock(spec_set=[ + "annotate_state_groups", + ]) + + self.auth = NonCallableMock(spec_set=[ + "check", + ]) + self.hostname = "test" hs = HomeServer( self.hostname, @@ -53,6 +60,8 @@ class FederationTestCase(unittest.TestCase): "federation_handler", ]), config=self.mock_config, + auth=self.auth, + state_handler=self.state_handler, ) self.datastore = hs.get_datastore() @@ -65,74 +74,35 @@ class FederationTestCase(unittest.TestCase): @defer.inlineCallbacks def test_msg(self): pdu = Pdu( - pdu_type=MessageEvent.TYPE, - context="foo", + type=MessageEvent.TYPE, + room_id="foo", content={"msgtype": u"fooo"}, origin_server_ts=0, - pdu_id="a", + event_id="$a:b", origin="b", ) - store_id = "ASD" - self.datastore.persist_event.return_value = defer.succeed(store_id) + self.datastore.persist_event.return_value = defer.succeed(None) self.datastore.get_room.return_value = defer.succeed(True) + self.state_handler.annotate_state_groups.return_value = ( + defer.succeed(False) + ) + yield self.handlers.federation_handler.on_receive_pdu(pdu, False) self.datastore.persist_event.assert_called_once_with( ANY, False, is_new_state=False ) - self.notifier.on_new_room_event.assert_called_once_with(ANY, extra_users=[]) - @defer.inlineCallbacks - def test_invite_join_target_this(self): - room_id = "foo" - user_id = "@bob:red" - - pdu = Pdu( - pdu_type=InviteJoinEvent.TYPE, - user_id=user_id, - target_host=self.hostname, - context=room_id, - content={}, - origin_server_ts=0, - pdu_id="a", - origin="b", + self.state_handler.annotate_state_groups.assert_called_once_with( + ANY, + old_state=None, ) - yield self.handlers.federation_handler.on_receive_pdu(pdu, False) + self.auth.check.assert_called_once_with(ANY, None, raises=True) - mem_handler = self.handlers.room_member_handler - self.assertEquals(1, mem_handler.change_membership.call_count) - call_args = get_mock_call_args( - lambda event, do_auth: None, - mem_handler.change_membership + self.notifier.on_new_room_event.assert_called_once_with( + ANY, + extra_users=[] ) - self.assertEquals(False, call_args["do_auth"]) - - new_event = call_args["event"] - self.assertEquals(RoomMemberEvent.TYPE, new_event.type) - self.assertEquals(room_id, new_event.room_id) - self.assertEquals(user_id, new_event.state_key) - self.assertEquals(Membership.JOIN, new_event.membership) - - @defer.inlineCallbacks - def test_invite_join_target_other(self): - room_id = "foo" - user_id = "@bob:red" - - pdu = Pdu( - pdu_type=InviteJoinEvent.TYPE, - user_id=user_id, - state_key="@red:not%s" % self.hostname, - context=room_id, - content={}, - origin_server_ts=0, - pdu_id="a", - origin="b", - ) - - yield self.handlers.federation_handler.on_receive_pdu(pdu, False) - - mem_handler = self.handlers.room_member_handler - self.assertEquals(0, mem_handler.change_membership.call_count) From dfb3d21a6d2560668a637bba149bf34c7d413125 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 17:12:39 +0000 Subject: [PATCH 070/227] Fix room handler tests --- synapse/handlers/room.py | 1 - tests/handlers/test_room.py | 146 +++++------------------------------- 2 files changed, 19 insertions(+), 128 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index ffc0892f1a..f176ad39bf 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -408,7 +408,6 @@ class RoomMemberHandler(BaseHandler): defer.returnValue({}) return - yield self.state_handler.handle_new_event(event, snapshot) yield self._do_local_membership_update( event, membership=event.content["membership"], diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index c88d1c8840..57a8a1b13d 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -18,7 +18,7 @@ from twisted.internet import defer from tests import unittest from synapse.api.events.room import ( - InviteJoinEvent, RoomMemberEvent, RoomConfigEvent + RoomMemberEvent, ) from synapse.api.constants import Membership from synapse.handlers.room import RoomMemberHandler, RoomCreationHandler @@ -34,6 +34,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): def setUp(self): self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] + self.hostname = "red" hs = HomeServer( self.hostname, @@ -58,7 +59,10 @@ class RoomMemberHandlerTestCase(unittest.TestCase): "federation_handler", ]), auth=NonCallableMock(spec_set=["check"]), - state_handler=NonCallableMock(spec_set=["handle_new_event"]), + state_handler=NonCallableMock(spec_set=[ + "handle_new_event", + "annotate_state_groups", + ]), config=self.mock_config, ) @@ -114,6 +118,8 @@ class RoomMemberHandlerTestCase(unittest.TestCase): store_id = "store_id_fooo" self.datastore.persist_event.return_value = defer.succeed(store_id) + self.datastore.get_room_member.return_value = defer.succeed(None) + # Actual invocation yield self.room_member_handler.change_membership(event) @@ -202,133 +208,16 @@ class RoomMemberHandlerTestCase(unittest.TestCase): join_signal_observer.assert_called_with( user=user, room_id=room_id) - @defer.inlineCallbacks - def STALE_test_invite_join(self): - room_id = "foo" - user_id = "@bob:red" - target_user_id = "@bob:red" - content = {"membership": Membership.JOIN} - - event = self.hs.get_event_factory().create_event( - etype=RoomMemberEvent.TYPE, - user_id=user_id, - target_user_id=target_user_id, - room_id=room_id, - membership=Membership.JOIN, - content=content, - ) - - joined = ["red", "blue", "green"] - - self.state_handler.handle_new_event.return_value = defer.succeed(True) - self.datastore.get_joined_hosts_for_room.return_value = ( - defer.succeed(joined) - ) - - store_id = "store_id_fooo" - self.datastore.store_room_member.return_value = defer.succeed(store_id) - self.datastore.get_room.return_value = defer.succeed(None) - - prev_state = NonCallableMock(name="prev_state") - prev_state.membership = Membership.INVITE - prev_state.sender = "@foo:blue" - self.datastore.get_room_member.return_value = defer.succeed(prev_state) - - # Actual invocation - yield self.room_member_handler.change_membership(event) - - self.datastore.get_room_member.assert_called_once_with( - target_user_id, room_id - ) - - self.assertTrue(self.federation.handle_new_event.called) - args = self.federation.handle_new_event.call_args[0] - invite_join_event = args[0] - - self.assertTrue(InviteJoinEvent.TYPE, invite_join_event.TYPE) - self.assertTrue("blue", invite_join_event.target_host) - self.assertTrue(room_id, invite_join_event.room_id) - self.assertTrue(user_id, invite_join_event.user_id) - self.assertFalse(hasattr(invite_join_event, "state_key")) - - self.assertEquals( - set(["blue"]), - set(invite_join_event.destinations) - ) - - self.federation.get_state_for_room.assert_called_once_with( - "blue", room_id - ) - - self.assertFalse(self.datastore.store_room_member.called) - - self.assertFalse(self.notifier.on_new_room_event.called) - self.assertFalse(self.state_handler.handle_new_event.called) - - @defer.inlineCallbacks - def STALE_test_invite_join_public(self): - room_id = "#foo:blue" - user_id = "@bob:red" - target_user_id = "@bob:red" - content = {"membership": Membership.JOIN} - - event = self.hs.get_event_factory().create_event( - etype=RoomMemberEvent.TYPE, - user_id=user_id, - target_user_id=target_user_id, - room_id=room_id, - membership=Membership.JOIN, - content=content, - ) - - joined = ["red", "blue", "green"] - - self.state_handler.handle_new_event.return_value = defer.succeed(True) - self.datastore.get_joined_hosts_for_room.return_value = ( - defer.succeed(joined) - ) - - store_id = "store_id_fooo" - self.datastore.store_room_member.return_value = defer.succeed(store_id) - self.datastore.get_room.return_value = defer.succeed(None) - - prev_state = NonCallableMock(name="prev_state") - prev_state.membership = Membership.INVITE - prev_state.sender = "@foo:blue" - self.datastore.get_room_member.return_value = defer.succeed(prev_state) - - # Actual invocation - yield self.room_member_handler.change_membership(event) - - self.assertTrue(self.federation.handle_new_event.called) - args = self.federation.handle_new_event.call_args[0] - invite_join_event = args[0] - - self.assertTrue(InviteJoinEvent.TYPE, invite_join_event.TYPE) - self.assertTrue("blue", invite_join_event.target_host) - self.assertTrue("foo", invite_join_event.room_id) - self.assertTrue(user_id, invite_join_event.user_id) - self.assertFalse(hasattr(invite_join_event, "state_key")) - - self.assertEquals( - set(["blue"]), - set(invite_join_event.destinations) - ) - - self.federation.get_state_for_room.assert_called_once_with( - "blue", "foo" - ) - - self.assertFalse(self.datastore.store_room_member.called) - - self.assertFalse(self.notifier.on_new_room_event.called) - self.assertFalse(self.state_handler.handle_new_event.called) - class RoomCreationTest(unittest.TestCase): def setUp(self): self.hostname = "red" + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + + hs = HomeServer( self.hostname, db_pool=None, @@ -346,11 +235,14 @@ class RoomCreationTest(unittest.TestCase): "federation_handler", ]), auth=NonCallableMock(spec_set=["check"]), - state_handler=NonCallableMock(spec_set=["handle_new_event"]), + state_handler=NonCallableMock(spec_set=[ + "handle_new_event", + "annotate_state_groups", + ]), ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.federation = NonCallableMock(spec_set=[ @@ -400,6 +292,6 @@ class RoomCreationTest(unittest.TestCase): self.assertEquals(user_id, join_event.user_id) self.assertEquals(user_id, join_event.state_key) - self.assertTrue(self.state_handler.handle_new_event.called) + self.assertTrue(self.state_handler.annotate_state_groups.called) self.assertTrue(self.federation.handle_new_event.called) From 4a5e95511e31aa1836259e94d501a9cfe359d466 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 17:13:21 +0000 Subject: [PATCH 071/227] PEP8 --- tests/handlers/test_room.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 57a8a1b13d..f0b77a2496 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -171,7 +171,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.datastore.get_joined_hosts_for_room.side_effect = get_joined - store_id = "store_id_fooo" self.datastore.persist_event.return_value = defer.succeed(store_id) self.datastore.get_room.return_value = defer.succeed(1) # Not None. @@ -203,10 +202,12 @@ class RoomMemberHandlerTestCase(unittest.TestCase): event ) self.notifier.on_new_room_event.assert_called_once_with( - event, extra_users=[user]) + event, extra_users=[user] + ) join_signal_observer.assert_called_with( - user=user, room_id=room_id) + user=user, room_id=room_id + ) class RoomCreationTest(unittest.TestCase): @@ -217,7 +218,6 @@ class RoomCreationTest(unittest.TestCase): self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] - hs = HomeServer( self.hostname, db_pool=None, From 96c001e6688617cc365f548a3152a32c647ebc59 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:07:54 +0000 Subject: [PATCH 072/227] Fix auth checks to all use the given old_event_state --- synapse/api/auth.py | 53 +++++++++++++++++++--------------- synapse/handlers/_base.py | 19 ++++++++---- synapse/handlers/federation.py | 6 ++-- synapse/state.py | 8 +++-- 4 files changed, 53 insertions(+), 33 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index c684265101..9eb0491c97 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -22,7 +22,7 @@ from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.api.events.room import ( RoomMemberEvent, RoomPowerLevelsEvent, RoomRedactionEvent, RoomJoinRulesEvent, RoomOpsPowerLevelsEvent, InviteJoinEvent, - RoomCreateEvent, + RoomCreateEvent, RoomSendEventLevelEvent, RoomAddStateLevelEvent, ) from synapse.util.logutils import log_function @@ -37,8 +37,7 @@ class Auth(object): self.hs = hs self.store = hs.get_datastore() - @defer.inlineCallbacks - def check(self, event, snapshot, raises=False): + def check(self, event, raises=False): """ Checks if this event is correctly authed. Returns: @@ -52,17 +51,17 @@ class Auth(object): if event.old_state_events is None: # Oh, we don't know what the state of the room was, so we # are trusting that this is allowed (at least for now) - defer.returnValue(True) + return True if hasattr(event, "outlier") and event.outlier is True: # TODO (erikj): Auth for outliers is done differently. - defer.returnValue(True) + return True is_state = hasattr(event, "state_key") if event.type == RoomCreateEvent.TYPE: # FIXME - defer.returnValue(True) + return True if event.type == RoomMemberEvent.TYPE: self._can_replace_state(event) @@ -71,8 +70,7 @@ class Auth(object): logger.debug("Allowing! %s", event) else: logger.debug("Denying! %s", event) - defer.returnValue(allowed) - return + return allowed if not event.type == InviteJoinEvent.TYPE: self.check_event_sender_in_room(event) @@ -80,10 +78,10 @@ class Auth(object): if is_state: # TODO (erikj): This really only should be called for *new* # state - yield self._can_add_state(event) + self._can_add_state(event) self._can_replace_state(event) else: - yield self._can_send_event(event) + self._can_send_event(event) if event.type == RoomPowerLevelsEvent.TYPE: self._check_power_levels(event) @@ -91,9 +89,8 @@ class Auth(object): if event.type == RoomRedactionEvent.TYPE: self._check_redaction(event) - logger.debug("Allowing! %s", event) - defer.returnValue(True) + return True else: raise AuthError(500, "Unknown event: %s" % event) except AuthError as e: @@ -103,7 +100,7 @@ class Auth(object): if raises: raise e - defer.returnValue(False) + return False @defer.inlineCallbacks def check_joined_room(self, room_id, user_id): @@ -326,10 +323,15 @@ class Auth(object): def is_server_admin(self, user): return self.store.is_server_admin(user) - @defer.inlineCallbacks @log_function def _can_send_event(self, event): - send_level = yield self.store.get_send_event_level(event.room_id) + key = (RoomSendEventLevelEvent.TYPE, "", ) + send_level_event = event.old_state_events.get(key) + send_level = None + if send_level_event: + send_level = send_level_event.content.get(event.user_id) + if not send_level: + send_level = send_level_event.content.get("level", 0) if send_level: send_level = int(send_level) @@ -351,16 +353,21 @@ class Auth(object): 403, "You don't have permission to post to the room" ) - defer.returnValue(True) + return True - @defer.inlineCallbacks def _can_add_state(self, event): - add_level = yield self.store.get_add_state_level(event.room_id) + key = (RoomAddStateLevelEvent.TYPE, "", ) + add_level_event = event.old_state_events.get(key) + add_level = None + if add_level_event: + add_level = add_level_event.content.get(event.user_id) + if not add_level: + add_level = add_level_event.content.get("level", 0) - if not add_level: - defer.returnValue(True) - - add_level = int(add_level) + if add_level: + add_level = int(add_level) + else: + add_level = 0 user_level = self._get_power_level_from_event_state( event, @@ -374,7 +381,7 @@ class Auth(object): 403, "You don't have permission to add state to the room" ) - defer.returnValue(True) + return True def _can_replace_state(self, event): user_level = self._get_power_level_from_event_state( diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 28b64565ae..509f7b550c 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -20,6 +20,12 @@ from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import add_hashes_and_signatures +import logging + + +logger = logging.getLogger(__name__) + + class BaseHandler(object): def __init__(self, hs): @@ -58,15 +64,18 @@ class BaseHandler(object): yield self.state_handler.annotate_state_groups(event) - yield add_hashes_and_signatures( + logger.debug("Signing event...") + + add_hashes_and_signatures( event, self.server_name, self.signing_key ) - if not suppress_auth: - yield self.auth.check(event, snapshot, raises=True) + logger.debug("Signed event.") - if hasattr(event, "state_key"): - yield self.state_handler.handle_new_event(event, snapshot) + if not suppress_auth: + logger.debug("Authing...") + self.auth.check(event, raises=True) + logger.debug("Authed") yield self.store.persist_event(event) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 06a2dabae2..1464a60937 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -118,7 +118,7 @@ class FederationHandler(BaseHandler): logger.debug("Event: %s", event) try: - yield self.auth.check(event, None, raises=True) + self.auth.check(event, raises=True) except AuthError as e: raise FederationError( "ERROR", @@ -319,7 +319,7 @@ class FederationHandler(BaseHandler): snapshot.fill_out_prev_events(event) yield self.state_handler.annotate_state_groups(event) - yield self.auth.check(event, None, raises=True) + self.auth.check(event, raises=True) pdu = self.pdu_codec.pdu_from_event(event) @@ -333,7 +333,7 @@ class FederationHandler(BaseHandler): event.outlier = False is_new_state = yield self.state_handler.annotate_state_groups(event) - yield self.auth.check(event, None, raises=True) + self.auth.check(event, raises=True) # FIXME (erikj): All this is duplicated above :( diff --git a/synapse/state.py b/synapse/state.py index 9771883bc3..32744e047c 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -188,11 +188,15 @@ class StateHandler(object): consumeErrors=True ) - max_power = max([int(p) for p in new_powers]) + new_powers = [ + int(p) if p else 0 for p in new_powers + ] + + max_power = max(new_powers) curr_events = [ z[0] for z in zip(curr_events, new_powers) - if int(z[1]) == max_power + if z[1] == max_power ] if not curr_events: From 9e429239ab0c304cce4d2f076be5b30691fd93a7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:10:36 +0000 Subject: [PATCH 073/227] Fix Federation test --- tests/handlers/test_federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 63a9990f7a..a9d6b2bb17 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -100,7 +100,7 @@ class FederationTestCase(unittest.TestCase): old_state=None, ) - self.auth.check.assert_called_once_with(ANY, None, raises=True) + self.auth.check.assert_called_once_with(ANY, raises=True) self.notifier.on_new_room_event.assert_called_once_with( ANY, From 34c4614682b08fa89a9f94ded4ecaf882fdc4160 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:10:54 +0000 Subject: [PATCH 074/227] Fix presence test --- tests/handlers/test_presence.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index fdc2e8de4a..a6af648def 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -51,6 +51,7 @@ def _expect_edu(destination, edu_type, content, origin="test"): "content": content, } ], + "pdu_failures": [], } def _make_edu_json(origin, edu_type, content): From 416ab4ebf0cd12a14f9d6d98226ae52d7d86f3a1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:12:47 +0000 Subject: [PATCH 075/227] Don't execute empty tests. Formatting. --- tests/rest/test_events.py | 45 ++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/tests/rest/test_events.py b/tests/rest/test_events.py index 362c7bc01c..82f5d277e6 100644 --- a/tests/rest/test_events.py +++ b/tests/rest/test_events.py @@ -25,9 +25,6 @@ import synapse.rest.room from synapse.server import HomeServer -# python imports -import json - from ..utils import MockHttpResource, MemoryDataStore, MockKey from .utils import RestTestCase @@ -49,7 +46,7 @@ class EventStreamPaginationApiTestCase(unittest.TestCase): def tearDown(self): pass - def test_long_poll(self): + def TODO_test_long_poll(self): # stream from 'end' key, send (self+other) message, expect message. # stream from 'END', send (self+other) message, expect message. @@ -64,7 +61,7 @@ class EventStreamPaginationApiTestCase(unittest.TestCase): pass - def test_stream_forward(self): + def TODO_test_stream_forward(self): # stream from START, expect injected items # stream from 'start' key, expect same content @@ -80,14 +77,14 @@ class EventStreamPaginationApiTestCase(unittest.TestCase): # returned as end key pass - def test_limits(self): + def TODO_test_limits(self): # stream from a key, expect limit_num items # stream from START, expect limit_num items pass - def test_range(self): + def TODO_test_range(self): # stream from key to key, expect X items # stream from key to END, expect X items @@ -97,7 +94,7 @@ class EventStreamPaginationApiTestCase(unittest.TestCase): # stream from START to END, expect all items pass - def test_direction(self): + def TODO_test_direction(self): # stream from END to START and fwds, expect newest first # stream from END to START and bwds, expect oldest first @@ -116,7 +113,10 @@ class EventStreamPermissionsTestCase(RestTestCase): def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - state_handler = Mock(spec=["handle_new_event"]) + state_handler = Mock(spec=[ + "handle_new_event", + "annotate_state_groups" + ]) state_handler.handle_new_event.return_value = True persistence_service = Mock(spec=["get_latest_pdus_in_context"]) @@ -151,6 +151,7 @@ class EventStreamPermissionsTestCase(RestTestCase): hs.get_handlers().federation_handler = Mock() hs.get_clock().time_msec.return_value = 1000000 + hs.get_clock().time.return_value = 1000 synapse.rest.register.register_servlets(hs, self.mock_resource) synapse.rest.events.register_servlets(hs, self.mock_resource) @@ -175,12 +176,14 @@ class EventStreamPermissionsTestCase(RestTestCase): def test_stream_basic_permissions(self): # invalid token, expect 403 (code, response) = yield self.mock_resource.trigger_get( - "/events?access_token=%s" % ("invalid" + self.token)) + "/events?access_token=%s" % ("invalid" + self.token, ) + ) self.assertEquals(403, code, msg=str(response)) # valid token, expect content (code, response) = yield self.mock_resource.trigger_get( - "/events?access_token=%s&timeout=0" % (self.token)) + "/events?access_token=%s&timeout=0" % (self.token,) + ) self.assertEquals(200, code, msg=str(response)) self.assertTrue("chunk" in response) self.assertTrue("start" in response) @@ -188,15 +191,23 @@ class EventStreamPermissionsTestCase(RestTestCase): @defer.inlineCallbacks def test_stream_room_permissions(self): - room_id = yield self.create_room_as(self.other_user, - tok=self.other_token) + room_id = yield self.create_room_as( + self.other_user, + tok=self.other_token + ) yield self.send(room_id, tok=self.other_token) # invited to room (expect no content for room) - yield self.invite(room_id, src=self.other_user, targ=self.user_id, - tok=self.other_token) + yield self.invite( + room_id, + src=self.other_user, + targ=self.user_id, + tok=self.other_token + ) + (code, response) = yield self.mock_resource.trigger_get( - "/events?access_token=%s&timeout=0" % (self.token)) + "/events?access_token=%s&timeout=0" % (self.token,) + ) self.assertEquals(200, code, msg=str(response)) self.assertEquals(0, len(response["chunk"])) @@ -206,7 +217,7 @@ class EventStreamPermissionsTestCase(RestTestCase): # left to room (expect no content for room) - def test_stream_items(self): + def TODO_test_stream_items(self): # new user, no content # join room, expect 1 item (join) From 24e2da455783009df1664cf94edf7653b80391a1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:13:01 +0000 Subject: [PATCH 076/227] Fix presencelike test --- tests/handlers/test_presencelike.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/handlers/test_presencelike.py b/tests/handlers/test_presencelike.py index 047752ad68..532ecf0f2c 100644 --- a/tests/handlers/test_presencelike.py +++ b/tests/handlers/test_presencelike.py @@ -21,7 +21,7 @@ from twisted.internet import defer from mock import Mock, call, ANY -from ..utils import MockClock +from ..utils import MockClock, MockKey from synapse.server import HomeServer from synapse.api.constants import PresenceState @@ -57,6 +57,9 @@ class PresenceAndProfileHandlers(object): class PresenceProfilelikeDataTestCase(unittest.TestCase): def setUp(self): + self.mock_config = Mock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", clock=MockClock(), db_pool=None, @@ -72,6 +75,7 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): resource_for_federation=Mock(), http_client=None, replication_layer=MockReplication(), + config=self.mock_config, ) hs.handlers = PresenceAndProfileHandlers(hs) From f48fce8bd37add27a43306eae255ca40b774a041 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:13:58 +0000 Subject: [PATCH 077/227] Fix profile tests --- tests/handlers/test_profile.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 5dc9b456e1..1660e7e928 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -24,7 +24,7 @@ from synapse.server import HomeServer from synapse.handlers.profile import ProfileHandler from synapse.api.constants import Membership -from tests.utils import SQLiteMemoryDbPool +from tests.utils import SQLiteMemoryDbPool, MockKey class ProfileHandlers(object): @@ -49,12 +49,16 @@ class ProfileTestCase(unittest.TestCase): db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() + self.mock_config = Mock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", db_pool=db_pool, http_client=None, handlers=None, resource_for_federation=Mock(), replication_layer=self.mock_federation, + config=self.mock_config, ) hs.handlers = ProfileHandlers(hs) From c670ce416bcbdeb929ce06dabcd4bce841ec815d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:15:15 +0000 Subject: [PATCH 078/227] Fix room tests --- tests/handlers/test_room.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index f0b77a2496..146ed21b65 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -123,9 +123,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase): # Actual invocation yield self.room_member_handler.change_membership(event) - self.state_handler.handle_new_event.assert_called_once_with( - event, self.snapshot, - ) self.federation.handle_new_event.assert_called_once_with( event, self.snapshot, ) @@ -186,9 +183,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase): # Actual invocation yield self.room_member_handler.change_membership(event) - self.state_handler.handle_new_event.assert_called_once_with( - event, self.snapshot - ) self.federation.handle_new_event.assert_called_once_with( event, self.snapshot ) From 1616df2f610ea2cc7c1fa32b98f5a63004d978de Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:15:56 +0000 Subject: [PATCH 079/227] Fix typing tests --- tests/handlers/test_typing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index f1d3b27f74..07acda5eee 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -40,6 +40,7 @@ def _expect_edu(destination, edu_type, content, origin="test"): "content": content, } ], + "pdu_failures": [], } From cc44ecc62f69436a9217745292af6c55b5f8fe81 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 13:23:35 +0000 Subject: [PATCH 080/227] Get correct prev_events --- synapse/storage/_base.py | 11 +++++------ synapse/storage/event_federation.py | 30 ++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 5e00c23fd1..7d445b4633 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -439,7 +439,7 @@ class SQLBaseStore(object): def _parse_events_txn(self, txn, rows): events = [self._parse_event_from_row(r) for r in rows] - sql = "SELECT * FROM events WHERE event_id = ?" + select_event_sql = "SELECT * FROM events WHERE event_id = ?" for ev in events: signatures = self._get_event_origin_signatures_txn( @@ -450,13 +450,12 @@ class SQLBaseStore(object): k: encode_base64(v) for k, v in signatures.items() } - prev_events = self._get_latest_events_in_room(txn, ev.room_id) - ev.prev_events = [(e_id, s,) for e_id, s, _ in prev_events] + ev.prev_events = self._get_prev_events(txn, ev.event_id) if hasattr(ev, "prev_state"): # Load previous state_content. # TODO: Should we be pulling this out above? - cursor = txn.execute(sql, (ev.prev_state,)) + cursor = txn.execute(select_event_sql, (ev.prev_state,)) prevs = self.cursor_to_dict(cursor) if prevs: prev = self._parse_event_from_row(prevs[0]) @@ -468,8 +467,8 @@ class SQLBaseStore(object): if ev.redacted: # Get the redaction event. - sql = "SELECT * FROM events WHERE event_id = ?" - txn.execute(sql, (ev.redacted,)) + select_event_sql = "SELECT * FROM events WHERE event_id = ?" + txn.execute(select_event_sql, (ev.redacted,)) del_evs = self._parse_events_txn( txn, self.cursor_to_dict(txn) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index dcc116bad2..f427aba879 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -49,15 +49,6 @@ class EventFederationStore(SQLBaseStore): ) def _get_latest_events_in_room(self, txn, room_id): - self._simple_select_onecol_txn( - txn, - table="event_forward_extremities", - keyvalues={ - "room_id": room_id, - }, - retcol="event_id", - ) - sql = ( "SELECT e.event_id, e.depth FROM events as e " "INNER JOIN event_forward_extremities as f " @@ -78,6 +69,27 @@ class EventFederationStore(SQLBaseStore): return results + def _get_prev_events(self, txn, event_id): + prev_ids = self._simple_select_onecol_txn( + txn, + table="event_edges", + keyvalues={ + "event_id": event_id, + }, + retcol="prev_event_id", + ) + + results = [] + for prev_event_id in prev_ids: + hashes = self._get_event_reference_hashes_txn(txn, prev_event_id) + prev_hashes = { + k: encode_base64(v) for k, v in hashes.items() + if k == "sha256" + } + results.append((event_id, prev_hashes)) + + return results + def get_min_depth(self, room_id): return self.runInteraction( "get_min_depth", From da80ebcc6b0bd15095d092eacd0ea55c81bb03b4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 13:27:20 +0000 Subject: [PATCH 081/227] Fix redaction storage test --- tests/storage/test_redaction.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index dae1641ea1..fb175a5c4c 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -61,6 +61,7 @@ class RedactionTestCase(unittest.TestCase): membership=membership, content={"membership": membership}, depth=self.depth, + prev_events=[], ) event.content.update(extra_content) @@ -68,6 +69,9 @@ class RedactionTestCase(unittest.TestCase): if prev_state: event.prev_state = prev_state + event.state_events = None + event.hashes = {} + # Have to create a join event using the eventfactory yield self.store.persist_event( event @@ -85,8 +89,12 @@ class RedactionTestCase(unittest.TestCase): room_id=room.to_string(), content={"body": body, "msgtype": u"message"}, depth=self.depth, + prev_events=[], ) + event.state_events = None + event.hashes = {} + yield self.store.persist_event( event ) @@ -102,8 +110,12 @@ class RedactionTestCase(unittest.TestCase): content={"reason": reason}, depth=self.depth, redacts=event_id, + prev_events=[], ) + event.state_events = None + event.hashes = {} + yield self.store.persist_event( event ) From 2fcce3b3c508ba1b10117e16511fb577da82f0b6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 13:43:36 +0000 Subject: [PATCH 082/227] Remove stale tests --- tests/storage/test_room.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index 369a73d917..4ff02c306b 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -127,7 +127,7 @@ class RoomEventsStoreTestCase(unittest.TestCase): ) @defer.inlineCallbacks - def test_room_name(self): + def STALE_test_room_name(self): name = u"A-Room-Name" yield self.inject_room_event( @@ -150,7 +150,7 @@ class RoomEventsStoreTestCase(unittest.TestCase): ) @defer.inlineCallbacks - def test_room_name(self): + def STALE_test_room_topic(self): topic = u"A place for things" yield self.inject_room_event( From 3791b75000f9787806dbdce3e6e216da1ac07388 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 13:48:36 +0000 Subject: [PATCH 083/227] Fix a couple more storage tests --- tests/storage/test_roommember.py | 24 +++++++++++++++--------- tests/storage/test_stream.py | 24 +++++++++++++++++------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index eae278ee8d..9e2b2e31c8 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -51,16 +51,22 @@ class RoomMemberStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def inject_room_member(self, room, user, membership): # Have to create a join event using the eventfactory + event = self.event_factory.create_event( + etype=RoomMemberEvent.TYPE, + user_id=user.to_string(), + state_key=user.to_string(), + room_id=room.to_string(), + membership=membership, + content={"membership": membership}, + depth=1, + prev_events=[], + ) + + event.state_events = None + event.hashes = {} + yield self.store.persist_event( - self.event_factory.create_event( - etype=RoomMemberEvent.TYPE, - user_id=user.to_string(), - state_key=user.to_string(), - room_id=room.to_string(), - membership=membership, - content={"membership": membership}, - depth=1, - ) + event ) @defer.inlineCallbacks diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index ab30e6ea25..b034fd8f16 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -59,8 +59,12 @@ class StreamStoreTestCase(unittest.TestCase): membership=membership, content={"membership": membership}, depth=self.depth, + prev_events=[], ) + event.state_events = None + event.hashes = {} + if prev_state: event.prev_state = prev_state @@ -75,15 +79,21 @@ class StreamStoreTestCase(unittest.TestCase): def inject_message(self, room, user, body): self.depth += 1 + event = self.event_factory.create_event( + etype=MessageEvent.TYPE, + user_id=user.to_string(), + room_id=room.to_string(), + content={"body": body, "msgtype": u"message"}, + depth=self.depth, + prev_events=[], + ) + + event.state_events = None + event.hashes = {} + # Have to create a join event using the eventfactory yield self.store.persist_event( - self.event_factory.create_event( - etype=MessageEvent.TYPE, - user_id=user.to_string(), - room_id=room.to_string(), - content={"body": body, "msgtype": u"message"}, - depth=self.depth, - ) + event ) @defer.inlineCallbacks From 4317c8e5835f0c15bf882f737d3e3c2a5b85f73f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Nov 2014 15:10:55 +0000 Subject: [PATCH 084/227] Implement new replace_state and changed prev_state `prev_state` is now a list of previous state ids, similiar to prev_events. `replace_state` now points to what we think was replaced. --- synapse/api/events/__init__.py | 1 + synapse/handlers/directory.py | 5 +- synapse/handlers/federation.py | 4 +- synapse/handlers/message.py | 11 +-- synapse/handlers/profile.py | 6 +- synapse/handlers/room.py | 16 ++--- synapse/rest/room.py | 2 +- synapse/state.py | 39 ++--------- synapse/storage/__init__.py | 92 +++++++++++++++++++------- synapse/storage/_base.py | 64 +++++++++++++----- synapse/storage/event_federation.py | 64 ++++++++++++++++-- synapse/storage/schema/event_edges.sql | 40 +++++++---- synapse/util/jsonobject.py | 2 +- 13 files changed, 219 insertions(+), 127 deletions(-) diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 168b812311..fc3f350570 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -60,6 +60,7 @@ class SynapseEvent(JsonEncodedObject): "age_ts", "prev_content", "prev_state", + "replaces_state", "redacted_because", "origin_server_ts", ] diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 6e897e915d..164363cdc5 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -147,10 +147,7 @@ class DirectoryHandler(BaseHandler): content={"aliases": aliases}, ) - snapshot = yield self.store.snapshot_room( - room_id=room_id, - user_id=user_id, - ) + snapshot = yield self.store.snapshot_room(event) yield self._on_new_room_event( event, snapshot, extra_users=[user_id], suppress_auth=True diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 1464a60937..513ec9a5e3 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -313,9 +313,7 @@ class FederationHandler(BaseHandler): state_key=user_id, ) - snapshot = yield self.store.snapshot_room( - event.room_id, event.user_id, - ) + snapshot = yield self.store.snapshot_room(event) snapshot.fill_out_prev_events(event) yield self.state_handler.annotate_state_groups(event) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c6f6ab14d1..8394013df3 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -81,7 +81,7 @@ class MessageHandler(BaseHandler): user = self.hs.parse_userid(event.user_id) assert user.is_mine, "User must be our own: %s" % (user,) - snapshot = yield self.store.snapshot_room(event.room_id, event.user_id) + snapshot = yield self.store.snapshot_room(event) yield self._on_new_room_event( event, snapshot, suppress_auth=suppress_auth @@ -141,12 +141,7 @@ class MessageHandler(BaseHandler): SynapseError if something went wrong. """ - snapshot = yield self.store.snapshot_room( - event.room_id, - event.user_id, - state_type=event.type, - state_key=event.state_key, - ) + snapshot = yield self.store.snapshot_room(event) yield self._on_new_room_event(event, snapshot) @@ -214,7 +209,7 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def send_feedback(self, event): - snapshot = yield self.store.snapshot_room(event.room_id, event.user_id) + snapshot = yield self.store.snapshot_room(event) # store message in db yield self._on_new_room_event(event, snapshot) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 4cd0a06093..e47814483a 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -17,7 +17,6 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, AuthError, CodeMessageException from synapse.api.constants import Membership -from synapse.api.events.room import RoomMemberEvent from ._base import BaseHandler @@ -196,10 +195,7 @@ class ProfileHandler(BaseHandler): ) for j in joins: - snapshot = yield self.store.snapshot_room( - j.room_id, j.state_key, RoomMemberEvent.TYPE, - j.state_key - ) + snapshot = yield self.store.snapshot_room(j) content = { "membership": j.content["membership"], diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index f176ad39bf..55c893eb58 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -122,10 +122,7 @@ class RoomCreationHandler(BaseHandler): @defer.inlineCallbacks def handle_event(event): - snapshot = yield self.store.snapshot_room( - room_id=room_id, - user_id=user_id, - ) + snapshot = yield self.store.snapshot_room(event) logger.debug("Event: %s", event) @@ -364,10 +361,8 @@ class RoomMemberHandler(BaseHandler): """ target_user_id = event.state_key - snapshot = yield self.store.snapshot_room( - event.room_id, event.user_id, - RoomMemberEvent.TYPE, target_user_id - ) + snapshot = yield self.store.snapshot_room(event) + ## TODO(markjh): get prev state from snapshot. prev_state = yield self.store.get_room_member( target_user_id, event.room_id @@ -442,10 +437,7 @@ class RoomMemberHandler(BaseHandler): content=content, ) - snapshot = yield self.store.snapshot_room( - room_id, joinee.to_string(), RoomMemberEvent.TYPE, - joinee.to_string() - ) + snapshot = yield self.store.snapshot_room(new_event) yield self._do_join(new_event, snapshot, room_host=host, do_auth=True) diff --git a/synapse/rest/room.py b/synapse/rest/room.py index ec0ce78fda..997895dab0 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -138,7 +138,7 @@ class RoomStateEventRestServlet(RestServlet): raise SynapseError( 404, "Event not found.", errcode=Codes.NOT_FOUND ) - defer.returnValue((200, data[0].get_dict()["content"])) + defer.returnValue((200, data.get_dict()["content"])) @defer.inlineCallbacks def on_PUT(self, request, room_id, event_type, state_key): diff --git a/synapse/state.py b/synapse/state.py index 32744e047c..97a8160a33 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -45,40 +45,6 @@ class StateHandler(object): self.server_name = hs.hostname self.hs = hs - @defer.inlineCallbacks - @log_function - def handle_new_event(self, event, snapshot): - """ Given an event this works out if a) we have sufficient power level - to update the state and b) works out what the prev_state should be. - - Returns: - Deferred: Resolved with a boolean indicating if we successfully - updated the state. - - Raised: - AuthError - """ - # This needs to be done in a transaction. - - if not hasattr(event, "state_key"): - return - - # Now I need to fill out the prev state and work out if it has auth - # (w.r.t. to power levels) - - snapshot.fill_out_prev_events(event) - yield self.annotate_state_groups(event) - - if event.old_state_events: - current_state = event.old_state_events.get( - (event.type, event.state_key) - ) - - if current_state: - event.prev_state = current_state.event_id - - defer.returnValue(True) - @defer.inlineCallbacks @log_function def annotate_state_groups(self, event, old_state=None): @@ -111,7 +77,10 @@ class StateHandler(object): event.old_state_events = copy.deepcopy(new_state) if hasattr(event, "state_key"): - new_state[(event.type, event.state_key)] = event + key = (event.type, event.state_key) + if key in new_state: + event.replaces_state = new_state[key].event_id + new_state[key] = event event.state_group = None event.state_events = new_state diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6b8fed4502..2d62fc2ed0 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -242,8 +242,8 @@ class DataStore(RoomMemberStore, RoomStore, "state_key": event.state_key, } - if hasattr(event, "prev_state"): - vals["prev_state"] = event.prev_state + if hasattr(event, "replaces_state"): + vals["prev_state"] = event.replaces_state self._simple_insert_txn(txn, "state_events", vals) @@ -258,6 +258,40 @@ class DataStore(RoomMemberStore, RoomStore, } ) + for e_id, h in event.prev_state: + self._simple_insert_txn( + txn, + table="event_edges", + values={ + "event_id": event.event_id, + "prev_event_id": e_id, + "room_id": event.room_id, + "is_state": 1, + }, + or_ignore=True, + ) + + if not backfilled: + self._simple_insert_txn( + txn, + table="state_forward_extremities", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + } + ) + + for prev_state_id, _ in event.prev_state: + self._simple_delete_txn( + txn, + table="state_forward_extremities", + keyvalues={ + "event_id": prev_state_id, + } + ) + for hash_alg, hash_base64 in event.hashes.items(): hash_bytes = decode_base64(hash_base64) self._store_event_content_hash_txn( @@ -357,7 +391,7 @@ class DataStore(RoomMemberStore, RoomStore, ], ) - def snapshot_room(self, room_id, user_id, state_type=None, state_key=None): + def snapshot_room(self, event): """Snapshot the room for an update by a user Args: room_id (synapse.types.RoomId): The room to snapshot. @@ -368,16 +402,29 @@ class DataStore(RoomMemberStore, RoomStore, synapse.storage.Snapshot: A snapshot of the state of the room. """ def _snapshot(txn): - membership_state = self._get_room_member(txn, user_id, room_id) - prev_events = self._get_latest_events_in_room(txn, room_id) + prev_events = self._get_latest_events_in_room( + txn, + event.room_id + ) + + prev_state = None + state_key = None + if hasattr(event, "state_key"): + state_key = event.state_key + prev_state = self._get_latest_state_in_room( + txn, + event.room_id, + type=event.type, + state_key=state_key, + ) return Snapshot( store=self, - room_id=room_id, - user_id=user_id, + room_id=event.room_id, + user_id=event.user_id, prev_events=prev_events, - membership_state=membership_state, - state_type=state_type, + prev_state=prev_state, + state_type=event.type, state_key=state_key, ) @@ -400,30 +447,29 @@ class Snapshot(object): """ def __init__(self, store, room_id, user_id, prev_events, - membership_state, state_type=None, state_key=None, - prev_state_pdu=None): + prev_state, state_type=None, state_key=None): self.store = store self.room_id = room_id self.user_id = user_id self.prev_events = prev_events - self.membership_state = membership_state + self.prev_state = prev_state self.state_type = state_type self.state_key = state_key - self.prev_state_pdu = prev_state_pdu def fill_out_prev_events(self, event): - if hasattr(event, "prev_events"): - return + if not hasattr(event, "prev_events"): + event.prev_events = [ + (event_id, hashes) + for event_id, hashes, _ in self.prev_events + ] - event.prev_events = [ - (event_id, hashes) - for event_id, hashes, _ in self.prev_events - ] + if self.prev_events: + event.depth = max([int(v) for _, _, v in self.prev_events]) + 1 + else: + event.depth = 0 - if self.prev_events: - event.depth = max([int(v) for _, _, v in self.prev_events]) + 1 - else: - event.depth = 0 + if not hasattr(event, "prev_state") and self.prev_state is not None: + event.prev_state = self.prev_state def schema_path(schema): diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 7d445b4633..7821fc4726 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -245,7 +245,6 @@ class SQLBaseStore(object): return [r[0] for r in txn.fetchall()] - def _simple_select_onecol(self, table, keyvalues, retcol): """Executes a SELECT query on the named table, which returns a list comprising of the values of the named column from the selected rows. @@ -273,17 +272,30 @@ class SQLBaseStore(object): keyvalues : dict of column names and values to select the rows with retcols : list of strings giving the names of the columns to return """ + return self.runInteraction( + "_simple_select_list", + self._simple_select_list_txn, + table, keyvalues, retcols + ) + + def _simple_select_list_txn(self, txn, table, keyvalues, retcols): + """Executes a SELECT query on the named table, which may return zero or + more rows, returning the result as a list of dicts. + + Args: + txn : Transaction object + table : string giving the table name + keyvalues : dict of column names and values to select the rows with + retcols : list of strings giving the names of the columns to return + """ sql = "SELECT %s FROM %s WHERE %s" % ( ", ".join(retcols), table, - " AND ".join("%s = ?" % (k) for k in keyvalues) + " AND ".join("%s = ?" % (k, ) for k in keyvalues) ) - def func(txn): - txn.execute(sql, keyvalues.values()) - return self.cursor_to_dict(txn) - - return self.runInteraction("_simple_select_list", func) + txn.execute(sql, keyvalues.values()) + return self.cursor_to_dict(txn) def _simple_update_one(self, table, keyvalues, updatevalues, retcols=None): @@ -417,6 +429,10 @@ class SQLBaseStore(object): d.pop("topological_ordering", None) d.pop("processed", None) d["origin_server_ts"] = d.pop("ts", 0) + replaces_state = d.pop("prev_state", None) + + if replaces_state: + d["replaces_state"] = replaces_state d.update(json.loads(row_dict["unrecognized_keys"])) d["content"] = json.loads(d["content"]) @@ -450,16 +466,32 @@ class SQLBaseStore(object): k: encode_base64(v) for k, v in signatures.items() } - ev.prev_events = self._get_prev_events(txn, ev.event_id) + prevs = self._get_prev_events_and_state(txn, ev.event_id) - if hasattr(ev, "prev_state"): - # Load previous state_content. - # TODO: Should we be pulling this out above? - cursor = txn.execute(select_event_sql, (ev.prev_state,)) - prevs = self.cursor_to_dict(cursor) - if prevs: - prev = self._parse_event_from_row(prevs[0]) - ev.prev_content = prev.content + ev.prev_events = [ + (e_id, h) + for e_id, h, is_state in prevs + if is_state == 0 + ] + + if hasattr(ev, "state_key"): + ev.prev_state = [ + (e_id, h) + for e_id, h, is_state in prevs + if is_state == 1 + ] + + if hasattr(ev, "replaces_state"): + # Load previous state_content. + # FIXME (erikj): Handle multiple prev_states. + cursor = txn.execute( + select_event_sql, + (ev.replaces_state,) + ) + prevs = self.cursor_to_dict(cursor) + if prevs: + prev = self._parse_event_from_row(prevs[0]) + ev.prev_content = prev.content if not hasattr(ev, "redacted"): logger.debug("Doesn't have redacted key: %s", ev) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index f427aba879..180a764134 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -69,19 +69,21 @@ class EventFederationStore(SQLBaseStore): return results - def _get_prev_events(self, txn, event_id): - prev_ids = self._simple_select_onecol_txn( + def _get_latest_state_in_room(self, txn, room_id, type, state_key): + event_ids = self._simple_select_onecol_txn( txn, - table="event_edges", + table="state_forward_extremities", keyvalues={ - "event_id": event_id, + "room_id": room_id, + "type": type, + "state_key": state_key, }, - retcol="prev_event_id", + retcol="event_id", ) results = [] - for prev_event_id in prev_ids: - hashes = self._get_event_reference_hashes_txn(txn, prev_event_id) + for event_id in event_ids: + hashes = self._get_event_reference_hashes_txn(txn, event_id) prev_hashes = { k: encode_base64(v) for k, v in hashes.items() if k == "sha256" @@ -90,6 +92,53 @@ class EventFederationStore(SQLBaseStore): return results + def _get_prev_events(self, txn, event_id): + results = self._get_prev_events_and_state( + txn, + event_id, + is_state=0, + ) + + return [(e_id, h, ) for e_id, h, _ in results] + + def _get_prev_state(self, txn, event_id): + results = self._get_prev_events_and_state( + txn, + event_id, + is_state=1, + ) + + return [(e_id, h, ) for e_id, h, _ in results] + + def _get_prev_events_and_state(self, txn, event_id, is_state=None): + keyvalues = { + "event_id": event_id, + } + + if is_state is not None: + keyvalues["is_state"] = is_state + + res = self._simple_select_list_txn( + txn, + table="event_edges", + keyvalues=keyvalues, + retcols=["prev_event_id", "is_state"], + ) + + results = [] + for d in res: + hashes = self._get_event_reference_hashes_txn( + txn, + d["prev_event_id"] + ) + prev_hashes = { + k: encode_base64(v) for k, v in hashes.items() + if k == "sha256" + } + results.append((d["prev_event_id"], prev_hashes, d["is_state"])) + + return results + def get_min_depth(self, room_id): return self.runInteraction( "get_min_depth", @@ -135,6 +184,7 @@ class EventFederationStore(SQLBaseStore): "event_id": event_id, "prev_event_id": e_id, "room_id": room_id, + "is_state": 0, }, or_ignore=True, ) diff --git a/synapse/storage/schema/event_edges.sql b/synapse/storage/schema/event_edges.sql index e5f768c705..51695826a8 100644 --- a/synapse/storage/schema/event_edges.sql +++ b/synapse/storage/schema/event_edges.sql @@ -1,7 +1,7 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( - event_id TEXT, - room_id TEXT, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE ); @@ -10,8 +10,8 @@ CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( - event_id TEXT, - room_id TEXT, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE ); @@ -20,10 +20,11 @@ CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id CREATE TABLE IF NOT EXISTS event_edges( - event_id TEXT, - prev_event_id TEXT, - room_id TEXT, - CONSTRAINT uniqueness UNIQUE (event_id, prev_event_id, room_id) + event_id TEXT NOT NULL, + prev_event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + is_state INTEGER NOT NULL, + CONSTRAINT uniqueness UNIQUE (event_id, prev_event_id, room_id, is_state) ); CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); @@ -31,8 +32,8 @@ CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( - room_id TEXT, - min_depth INTEGER, + room_id TEXT NOT NULL, + min_depth INTEGER NOT NULL, CONSTRAINT uniqueness UNIQUE (room_id) ); @@ -40,10 +41,25 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( - event_id TEXT, - destination TEXT, + event_id TEXT NOT NULL, + destination TEXT NOT NULL, delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered CONSTRAINT uniqueness UNIQUE (event_id, destination) ON CONFLICT REPLACE ); CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); + + +CREATE TABLE IF NOT EXISTS state_forward_extremities( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( + room_id, type, state_key +); +CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); + diff --git a/synapse/util/jsonobject.py b/synapse/util/jsonobject.py index c91eb897a8..e79b68f661 100644 --- a/synapse/util/jsonobject.py +++ b/synapse/util/jsonobject.py @@ -80,7 +80,7 @@ class JsonEncodedObject(object): def get_full_dict(self): d = { - k: v for (k, v) in self.__dict__.items() + k: _encode(v) for (k, v) in self.__dict__.items() if k in self.valid_keys or k in self.internal_keys } d.update(self.unrecognized_keys) From c6766d45b53ba51121ca15ae5806a5ac30f4f5d6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Nov 2014 15:19:00 +0000 Subject: [PATCH 085/227] Don't send prev_state to clients anymore --- synapse/api/events/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index fc3f350570..98a66144e7 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -59,7 +59,6 @@ class SynapseEvent(JsonEncodedObject): "required_power_level", "age_ts", "prev_content", - "prev_state", "replaces_state", "redacted_because", "origin_server_ts", @@ -76,6 +75,7 @@ class SynapseEvent(JsonEncodedObject): "prev_events", "hashes", "signatures", + "prev_state", ] required_keys = [ From 233969bb586f2a5b0231c8e0675c8c8c3097ef03 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Nov 2014 15:25:03 +0000 Subject: [PATCH 086/227] Update to use replaces_state rather than prev_state --- synapse/handlers/federation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 513ec9a5e3..f0448a05d8 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -386,8 +386,10 @@ class FederationHandler(BaseHandler): event = yield self.store.get_event(event_id) if hasattr(event, "state_key"): # Get previous state - if hasattr(event, "prev_state") and event.prev_state: - prev_event = yield self.store.get_event(event.prev_state) + if hasattr(event, "replaces_state") and event.replaces_state: + prev_event = yield self.store.get_event( + event.replaces_state + ) results[(event.type, event.state_key)] = prev_event else: del results[(event.type, event.state_key)] From 351c64e99e5677096f9a2ae2cd7e84dbc1887878 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Nov 2014 16:59:13 +0000 Subject: [PATCH 087/227] Amalgamate all power levels. Remove concept of reqired power levels, something similiar can be done using the new power level event. --- synapse/api/auth.py | 223 +++++++++++---------------------- synapse/api/events/__init__.py | 3 +- synapse/api/events/factory.py | 7 +- synapse/api/events/room.py | 22 ---- synapse/api/events/utils.py | 23 ++-- synapse/handlers/room.py | 52 +++----- synapse/storage/__init__.py | 12 -- synapse/storage/room.py | 157 ----------------------- 8 files changed, 103 insertions(+), 396 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 9eb0491c97..462e97bd90 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,8 +21,8 @@ from synapse.api.constants import Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.api.events.room import ( RoomMemberEvent, RoomPowerLevelsEvent, RoomRedactionEvent, - RoomJoinRulesEvent, RoomOpsPowerLevelsEvent, InviteJoinEvent, - RoomCreateEvent, RoomSendEventLevelEvent, RoomAddStateLevelEvent, + RoomJoinRulesEvent, InviteJoinEvent, + RoomCreateEvent, ) from synapse.util.logutils import log_function @@ -51,6 +51,7 @@ class Auth(object): if event.old_state_events is None: # Oh, we don't know what the state of the room was, so we # are trusting that this is allowed (at least for now) + logger.warn("Trusting event: %s", event.event_id) return True if hasattr(event, "outlier") and event.outlier is True: @@ -64,7 +65,7 @@ class Auth(object): return True if event.type == RoomMemberEvent.TYPE: - self._can_replace_state(event) + self._can_send_event(event) allowed = self.is_membership_change_allowed(event) if allowed: logger.debug("Allowing! %s", event) @@ -72,16 +73,7 @@ class Auth(object): logger.debug("Denying! %s", event) return allowed - if not event.type == InviteJoinEvent.TYPE: - self.check_event_sender_in_room(event) - - if is_state: - # TODO (erikj): This really only should be called for *new* - # state - self._can_add_state(event) - self._can_replace_state(event) - else: - self._can_send_event(event) + self._can_send_event(event) if event.type == RoomPowerLevelsEvent.TYPE: self._check_power_levels(event) @@ -239,21 +231,21 @@ class Auth(object): power_level_event = event.old_state_events.get(key) level = None if power_level_event: - level = power_level_event.content.get(user_id) + level = power_level_event.content.get("users", {}).get(user_id) if not level: - level = power_level_event.content.get("default", 0) + level = power_level_event.content.get("users_default", 0) return level def _get_ops_level_from_event_state(self, event): - key = (RoomOpsPowerLevelsEvent.TYPE, "", ) - ops_event = event.old_state_events.get(key) + key = (RoomPowerLevelsEvent.TYPE, "", ) + power_level_event = event.old_state_events.get(key) - if ops_event: + if power_level_event: return ( - ops_event.content.get("ban_level"), - ops_event.content.get("kick_level"), - ops_event.content.get("redact_level"), + power_level_event.content.get("ban", 50), + power_level_event.content.get("kick", 50), + power_level_event.content.get("redact", 50), ) return None, None, None, @@ -325,13 +317,22 @@ class Auth(object): @log_function def _can_send_event(self, event): - key = (RoomSendEventLevelEvent.TYPE, "", ) + key = (RoomPowerLevelsEvent.TYPE, "", ) send_level_event = event.old_state_events.get(key) send_level = None if send_level_event: - send_level = send_level_event.content.get(event.user_id) + send_level = send_level_event.content.get("events", {}).get( + event.type + ) if not send_level: - send_level = send_level_event.content.get("level", 0) + if hasattr(event, "state_key"): + send_level = send_level_event.content.get( + "state_default", 50 + ) + else: + send_level = send_level_event.content.get( + "events_default", 0 + ) if send_level: send_level = int(send_level) @@ -350,85 +351,21 @@ class Auth(object): if user_level < send_level: raise AuthError( - 403, "You don't have permission to post to the room" + 403, "You don't have permission to post that to the room" ) return True - def _can_add_state(self, event): - key = (RoomAddStateLevelEvent.TYPE, "", ) - add_level_event = event.old_state_events.get(key) - add_level = None - if add_level_event: - add_level = add_level_event.content.get(event.user_id) - if not add_level: - add_level = add_level_event.content.get("level", 0) - - if add_level: - add_level = int(add_level) - else: - add_level = 0 - - user_level = self._get_power_level_from_event_state( - event, - event.user_id, - ) - - user_level = int(user_level) - - if user_level < add_level: - raise AuthError( - 403, "You don't have permission to add state to the room" - ) - - return True - - def _can_replace_state(self, event): - user_level = self._get_power_level_from_event_state( - event, - event.user_id, - ) - - if user_level: - user_level = int(user_level) - else: - user_level = 0 - - logger.debug( - "Checking power level for %s, %s", event.user_id, user_level - ) - - key = (event.type, event.state_key, ) - current_state = event.old_state_events.get(key) - - if current_state and hasattr(current_state, "required_power_level"): - req = current_state.required_power_level - - logger.debug("Checked power level for %s, %s", event.user_id, req) - if user_level < req: - raise AuthError( - 403, - "You don't have permission to change that state" - ) - def _check_redaction(self, event): user_level = self._get_power_level_from_event_state( event, event.user_id, ) - if user_level: - user_level = int(user_level) - else: - user_level = 0 - _, _, redact_level = self._get_ops_level_from_event_state( event ) - if not redact_level: - redact_level = 50 - if user_level < redact_level: raise AuthError( 403, @@ -436,14 +373,9 @@ class Auth(object): ) def _check_power_levels(self, event): - for k, v in event.content.items(): - if k == "default": - continue - - # FIXME (erikj): We don't want hsob_Ts in content. - if k == "hsob_ts": - continue - + user_list = event.content.get("users", {}) + # Validate users + for k, v in user_list.items(): try: self.hs.parse_userid(k) except: @@ -459,72 +391,63 @@ class Auth(object): if not current_state: return - else: - current_state = current_state[0] user_level = self._get_power_level_from_event_state( event, event.user_id, ) - if user_level: - user_level = int(user_level) - else: - user_level = 0 + # Check other levels: + levels_to_check = [ + ("users_default", []), + ("events_default", []), + ("ban", []), + ("redact", []), + ("kick", []), + ] - old_list = current_state.content + old_list = current_state.content.get("users") + for user in set(old_list.keys() + user_list.keys()): + levels_to_check.append( + (user, ["users"]) + ) - # FIXME (erikj) - old_people = {k: v for k, v in old_list.items() if k.startswith("@")} - new_people = { - k: v for k, v in event.content.items() - if k.startswith("@") - } + old_list = current_state.content.get("events") + new_list = event.content.get("events") + for ev_id in set(old_list.keys() + new_list.keys()): + levels_to_check.append( + (ev_id, ["events"]) + ) - removed = set(old_people.keys()) - set(new_people.keys()) - added = set(new_people.keys()) - set(old_people.keys()) - same = set(old_people.keys()) & set(new_people.keys()) + old_state = current_state.content + new_state = event.content - for r in removed: - if int(old_list[r]) > user_level: - raise AuthError( - 403, - "You don't have permission to remove user: %s" % (r, ) - ) + for level_to_check, dir in levels_to_check: + old_loc = old_state + for d in dir: + old_loc = old_loc.get(d, {}) - for n in added: - if int(event.content[n]) > user_level: + new_loc = new_state + for d in dir: + new_loc = new_loc.get(d, {}) + + if level_to_check in old_loc: + old_level = int(old_loc[level_to_check]) + else: + old_level = None + + if level_to_check in new_loc: + new_level = int(new_loc[level_to_check]) + else: + new_level = None + + if new_level is not None and old_level is not None: + if new_level == old_level: + continue + + if old_level > user_level or new_level > user_level: raise AuthError( 403, "You don't have permission to add ops level greater " "than your own" ) - - for s in same: - if int(event.content[s]) != int(old_list[s]): - if int(event.content[s]) > user_level: - raise AuthError( - 403, - "You don't have permission to add ops level greater " - "than your own" - ) - - if "default" in old_list: - old_default = int(old_list["default"]) - - if old_default > user_level: - raise AuthError( - 403, - "You don't have permission to add ops level greater than " - "your own" - ) - - if "default" in event.content: - new_default = int(event.content["default"]) - - if new_default > user_level: - raise AuthError( - 403, - "You don't have permission to add ops level greater " - "than your own" - ) diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 98a66144e7..84d3a98365 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -56,12 +56,12 @@ class SynapseEvent(JsonEncodedObject): "user_id", # sender/initiator "content", # HTTP body, JSON "state_key", - "required_power_level", "age_ts", "prev_content", "replaces_state", "redacted_because", "origin_server_ts", + "auth_chains", ] internal_keys = [ @@ -70,7 +70,6 @@ class SynapseEvent(JsonEncodedObject): "destinations", "origin", "outlier", - "power_level", "redacted", "prev_events", "hashes", diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index 9134c82eff..a1ec708a81 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -16,8 +16,8 @@ from synapse.api.events.room import ( RoomTopicEvent, MessageEvent, RoomMemberEvent, FeedbackEvent, InviteJoinEvent, RoomConfigEvent, RoomNameEvent, GenericEvent, - RoomPowerLevelsEvent, RoomJoinRulesEvent, RoomOpsPowerLevelsEvent, - RoomCreateEvent, RoomAddStateLevelEvent, RoomSendEventLevelEvent, + RoomPowerLevelsEvent, RoomJoinRulesEvent, + RoomCreateEvent, RoomRedactionEvent, ) @@ -39,9 +39,6 @@ class EventFactory(object): RoomPowerLevelsEvent, RoomJoinRulesEvent, RoomCreateEvent, - RoomAddStateLevelEvent, - RoomSendEventLevelEvent, - RoomOpsPowerLevelsEvent, RoomRedactionEvent, ] diff --git a/synapse/api/events/room.py b/synapse/api/events/room.py index cd936074fc..25bc883706 100644 --- a/synapse/api/events/room.py +++ b/synapse/api/events/room.py @@ -153,28 +153,6 @@ class RoomPowerLevelsEvent(SynapseStateEvent): def get_content_template(self): return {} - -class RoomAddStateLevelEvent(SynapseStateEvent): - TYPE = "m.room.add_state_level" - - def get_content_template(self): - return {} - - -class RoomSendEventLevelEvent(SynapseStateEvent): - TYPE = "m.room.send_event_level" - - def get_content_template(self): - return {} - - -class RoomOpsPowerLevelsEvent(SynapseStateEvent): - TYPE = "m.room.ops_levels" - - def get_content_template(self): - return {} - - class RoomAliasesEvent(SynapseStateEvent): TYPE = "m.room.aliases" diff --git a/synapse/api/events/utils.py b/synapse/api/events/utils.py index 31601fd3a9..5fc79105b5 100644 --- a/synapse/api/events/utils.py +++ b/synapse/api/events/utils.py @@ -15,7 +15,6 @@ from .room import ( RoomMemberEvent, RoomJoinRulesEvent, RoomPowerLevelsEvent, - RoomAddStateLevelEvent, RoomSendEventLevelEvent, RoomOpsPowerLevelsEvent, RoomAliasesEvent, RoomCreateEvent, ) @@ -52,17 +51,17 @@ def _prune_event_or_pdu(event_type, event): elif event_type == RoomJoinRulesEvent.TYPE: add_fields("join_rule") elif event_type == RoomPowerLevelsEvent.TYPE: - # TODO: Actually check these are valid user_ids etc. - add_fields("default") - for k, v in event.content.items(): - if k.startswith("@") and isinstance(v, (int, long)): - new_content[k] = v - elif event_type == RoomAddStateLevelEvent.TYPE: - add_fields("level") - elif event_type == RoomSendEventLevelEvent.TYPE: - add_fields("level") - elif event_type == RoomOpsPowerLevelsEvent.TYPE: - add_fields("kick_level", "ban_level", "redact_level") + add_fields( + "users", + "users_default", + "events", + "events_default", + "events_default", + "state_default", + "ban", + "kick", + "redact", + ) elif event_type == RoomAliasesEvent.TYPE: add_fields("aliases") diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 55c893eb58..42a6c9f9bf 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -21,8 +21,7 @@ from synapse.api.constants import Membership, JoinRules from synapse.api.errors import StoreError, SynapseError from synapse.api.events.room import ( RoomMemberEvent, RoomCreateEvent, RoomPowerLevelsEvent, - RoomJoinRulesEvent, RoomAddStateLevelEvent, RoomTopicEvent, - RoomSendEventLevelEvent, RoomOpsPowerLevelsEvent, RoomNameEvent, + RoomTopicEvent, RoomNameEvent, RoomJoinRulesEvent, ) from synapse.util import stringutils from ._base import BaseHandler @@ -139,7 +138,6 @@ class RoomCreationHandler(BaseHandler): etype=RoomNameEvent.TYPE, room_id=room_id, user_id=user_id, - required_power_level=50, content={"name": name}, ) @@ -151,7 +149,6 @@ class RoomCreationHandler(BaseHandler): etype=RoomTopicEvent.TYPE, room_id=room_id, user_id=user_id, - required_power_level=50, content={"topic": topic}, ) @@ -196,7 +193,6 @@ class RoomCreationHandler(BaseHandler): event_keys = { "room_id": room_id, "user_id": creator.to_string(), - "required_power_level": 100, } def create(etype, **content): @@ -213,7 +209,21 @@ class RoomCreationHandler(BaseHandler): power_levels_event = self.event_factory.create_event( etype=RoomPowerLevelsEvent.TYPE, - content={creator.to_string(): 100, "default": 0}, + content={ + "users": { + creator.to_string(): 100, + }, + "users_default": 0, + "events": { + RoomNameEvent.TYPE: 100, + RoomPowerLevelsEvent.TYPE: 100, + }, + "events_default": 0, + "state_default": 50, + "ban": 50, + "kick": 50, + "redact": 50 + }, **event_keys ) @@ -223,30 +233,10 @@ class RoomCreationHandler(BaseHandler): join_rule=join_rule, ) - add_state_event = create( - etype=RoomAddStateLevelEvent.TYPE, - level=100, - ) - - send_event = create( - etype=RoomSendEventLevelEvent.TYPE, - level=0, - ) - - ops = create( - etype=RoomOpsPowerLevelsEvent.TYPE, - ban_level=50, - kick_level=50, - redact_level=50, - ) - return [ creation_event, power_levels_event, join_rules_event, - add_state_event, - send_event, - ops, ] @@ -388,16 +378,6 @@ class RoomMemberHandler(BaseHandler): else: # This is not a JOIN, so we can handle it normally. - # If we're banning someone, set a req power level - if event.membership == Membership.BAN: - if not hasattr(event, "required_power_level") or event.required_power_level is None: - # Add some default required_power_level - user_level = yield self.store.get_power_level( - event.room_id, - event.user_id, - ) - event.required_power_level = user_level - if prev_state and prev_state.membership == event.membership: # double same action, treat this event as a NOOP. defer.returnValue({}) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 2d62fc2ed0..2a1970914f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -17,13 +17,9 @@ from twisted.internet import defer from synapse.api.events.room import ( RoomMemberEvent, RoomTopicEvent, FeedbackEvent, -# RoomConfigEvent, RoomNameEvent, RoomJoinRulesEvent, RoomPowerLevelsEvent, - RoomAddStateLevelEvent, - RoomSendEventLevelEvent, - RoomOpsPowerLevelsEvent, RoomRedactionEvent, ) @@ -166,14 +162,6 @@ class DataStore(RoomMemberStore, RoomStore, self._store_room_topic_txn(txn, event) elif event.type == RoomJoinRulesEvent.TYPE: self._store_join_rule(txn, event) - elif event.type == RoomPowerLevelsEvent.TYPE: - self._store_power_levels(txn, event) - elif event.type == RoomAddStateLevelEvent.TYPE: - self._store_add_state_level(txn, event) - elif event.type == RoomSendEventLevelEvent.TYPE: - self._store_send_event_level(txn, event) - elif event.type == RoomOpsPowerLevelsEvent.TYPE: - self._store_ops_level(txn, event) elif event.type == RoomRedactionEvent.TYPE: self._store_redaction(txn, event) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 7e48ce9cc3..0c83c11ad3 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -148,85 +148,6 @@ class RoomStore(SQLBaseStore): else: defer.returnValue(None) - def get_power_level(self, room_id, user_id): - return self.runInteraction( - "get_power_level", - self._get_power_level, - room_id, user_id, - ) - - def _get_power_level(self, txn, room_id, user_id): - sql = ( - "SELECT level FROM room_power_levels as r " - "INNER JOIN current_state_events as c " - "ON r.event_id = c.event_id " - "WHERE c.room_id = ? AND r.user_id = ? " - ) - - rows = txn.execute(sql, (room_id, user_id,)).fetchall() - - if len(rows) == 1: - return rows[0][0] - - sql = ( - "SELECT level FROM room_default_levels as r " - "INNER JOIN current_state_events as c " - "ON r.event_id = c.event_id " - "WHERE c.room_id = ? " - ) - - rows = txn.execute(sql, (room_id,)).fetchall() - - if len(rows) == 1: - return rows[0][0] - else: - return None - - def get_ops_levels(self, room_id): - return self.runInteraction( - "get_ops_levels", - self._get_ops_levels, - room_id, - ) - - def _get_ops_levels(self, txn, room_id): - sql = ( - "SELECT ban_level, kick_level, redact_level " - "FROM room_ops_levels as r " - "INNER JOIN current_state_events as c " - "ON r.event_id = c.event_id " - "WHERE c.room_id = ? " - ) - - rows = txn.execute(sql, (room_id,)).fetchall() - - if len(rows) == 1: - return OpsLevel(rows[0][0], rows[0][1], rows[0][2]) - else: - return OpsLevel(None, None) - - def get_add_state_level(self, room_id): - return self._get_level_from_table("room_add_state_levels", room_id) - - def get_send_event_level(self, room_id): - return self._get_level_from_table("room_send_event_levels", room_id) - - @defer.inlineCallbacks - def _get_level_from_table(self, table, room_id): - sql = ( - "SELECT level FROM %(table)s as r " - "INNER JOIN current_state_events as c " - "ON r.event_id = c.event_id " - "WHERE c.room_id = ? " - ) % {"table": table} - - rows = yield self._execute(None, sql, room_id) - - if len(rows) == 1: - defer.returnValue(rows[0][0]) - else: - defer.returnValue(None) - def _store_room_topic_txn(self, txn, event): self._simple_insert_txn( txn, @@ -260,84 +181,6 @@ class RoomStore(SQLBaseStore): }, ) - def _store_power_levels(self, txn, event): - for user_id, level in event.content.items(): - if user_id == "default": - self._simple_insert_txn( - txn, - "room_default_levels", - { - "event_id": event.event_id, - "room_id": event.room_id, - "level": level, - }, - ) - else: - self._simple_insert_txn( - txn, - "room_power_levels", - { - "event_id": event.event_id, - "room_id": event.room_id, - "user_id": user_id, - "level": level - }, - ) - - def _store_default_level(self, txn, event): - self._simple_insert_txn( - txn, - "room_default_levels", - { - "event_id": event.event_id, - "room_id": event.room_id, - "level": event.content["default_level"], - }, - ) - - def _store_add_state_level(self, txn, event): - self._simple_insert_txn( - txn, - "room_add_state_levels", - { - "event_id": event.event_id, - "room_id": event.room_id, - "level": event.content["level"], - }, - ) - - def _store_send_event_level(self, txn, event): - self._simple_insert_txn( - txn, - "room_send_event_levels", - { - "event_id": event.event_id, - "room_id": event.room_id, - "level": event.content["level"], - }, - ) - - def _store_ops_level(self, txn, event): - content = { - "event_id": event.event_id, - "room_id": event.room_id, - } - - if "kick_level" in event.content: - content["kick_level"] = event.content["kick_level"] - - if "ban_level" in event.content: - content["ban_level"] = event.content["ban_level"] - - if "redact_level" in event.content: - content["redact_level"] = event.content["redact_level"] - - self._simple_insert_txn( - txn, - "room_ops_levels", - content, - ) - class RoomsTable(Table): table_name = "rooms" From 46de65cab9e449589e49b6cf7d37d4921363d934 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Nov 2014 17:15:09 +0000 Subject: [PATCH 088/227] Don't query the DB for user power levels --- synapse/state.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 97a8160a33..e2fd48bdae 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor +from synapse.api.events.room import RoomPowerLevelsEvent from collections import namedtuple @@ -141,21 +142,26 @@ class StateHandler(object): defer.returnValue(new_state) + def _get_power_level_from_event_state(self, event, user_id): + key = (RoomPowerLevelsEvent.TYPE, "", ) + power_level_event = event.old_state_events.get(key) + level = None + if power_level_event: + level = power_level_event.content.get("users", {}).get(user_id) + if not level: + level = power_level_event.content.get("users_default", 0) + + return level + @defer.inlineCallbacks @log_function def _resolve_state_events(self, events): curr_events = events - new_powers_deferreds = [] - for e in curr_events: - new_powers_deferreds.append( - self.store.get_power_level(e.room_id, e.user_id) - ) - - new_powers = yield defer.gatherResults( - new_powers_deferreds, - consumeErrors=True - ) + new_powers = [ + self._get_power_level_from_event_state(e, e.user_id) + for e in curr_events + ] new_powers = [ int(p) if p else 0 for p in new_powers From 8421cabb9db89a9d4d2158f1a4828b15c4ce18e9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Nov 2014 17:26:13 +0000 Subject: [PATCH 089/227] Neaten things up a bit --- synapse/api/auth.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 462e97bd90..bb25c4ec55 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -58,14 +58,13 @@ class Auth(object): # TODO (erikj): Auth for outliers is done differently. return True - is_state = hasattr(event, "state_key") - if event.type == RoomCreateEvent.TYPE: # FIXME return True + self._can_send_event(event) + if event.type == RoomMemberEvent.TYPE: - self._can_send_event(event) allowed = self.is_membership_change_allowed(event) if allowed: logger.debug("Allowing! %s", event) @@ -73,8 +72,6 @@ class Auth(object): logger.debug("Denying! %s", event) return allowed - self._can_send_event(event) - if event.type == RoomPowerLevelsEvent.TYPE: self._check_power_levels(event) From bf6b72eb558cca94e209a541188079750bfefea0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Nov 2014 18:42:18 +0000 Subject: [PATCH 090/227] Start implementing auth chains --- synapse/api/auth.py | 3 +- synapse/api/events/__init__.py | 2 +- synapse/handlers/_base.py | 59 +++++++++++++++++++++++++- synapse/storage/__init__.py | 12 +++++- synapse/storage/_base.py | 2 + synapse/storage/event_federation.py | 21 +++++++++ synapse/storage/schema/event_edges.sql | 10 +++++ synapse/storage/signatures.py | 12 ++++++ 8 files changed, 115 insertions(+), 6 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index bb25c4ec55..e1302553d7 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,8 +21,7 @@ from synapse.api.constants import Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.api.events.room import ( RoomMemberEvent, RoomPowerLevelsEvent, RoomRedactionEvent, - RoomJoinRulesEvent, InviteJoinEvent, - RoomCreateEvent, + RoomJoinRulesEvent, RoomCreateEvent, ) from synapse.util.logutils import log_function diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 84d3a98365..513a48f568 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -61,7 +61,7 @@ class SynapseEvent(JsonEncodedObject): "replaces_state", "redacted_because", "origin_server_ts", - "auth_chains", + "auth_events", ] internal_keys = [ diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 509f7b550c..2613fa7fce 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -14,11 +14,15 @@ # limitations under the License. from twisted.internet import defer + from synapse.api.errors import LimitExceededError - from synapse.util.async import run_on_reactor - from synapse.crypto.event_signing import add_hashes_and_signatures +from synapse.api.events.room import ( + RoomCreateEvent, RoomMemberEvent, RoomPowerLevelsEvent, RoomJoinRulesEvent, +) +from synapse.api.constants import Membership, JoinRules +from syutil.base64util import encode_base64 import logging @@ -55,6 +59,53 @@ class BaseHandler(object): retry_after_ms=int(1000*(time_allowed - time_now)), ) + @defer.inlineCallbacks + def _add_auth(self, event): + if event.type == RoomCreateEvent.TYPE: + event.auth_events = [] + return + + auth_events = [] + + key = (RoomPowerLevelsEvent.TYPE, "", ) + power_level_event = event.old_state_events.get(key) + + if power_level_event: + auth_events.append(power_level_event.event_id) + + key = (RoomJoinRulesEvent.TYPE, "", ) + join_rule_event = event.old_state_events.get(key) + + key = (RoomMemberEvent.TYPE, event.user_id, ) + member_event = event.old_state_events.get(key) + + if join_rule_event: + join_rule = join_rule_event.content.get("join_rule") + is_public = join_rule == JoinRules.PUBLIC if join_rule else False + + if event.type == RoomMemberEvent.TYPE: + if event.content["membership"] == Membership.JOIN: + if is_public: + auth_events.append(join_rule_event.event_id) + elif member_event: + auth_events.append(member_event.event_id) + + if member_event: + if member_event.content["membership"] == Membership.JOIN: + auth_events.append(member_event.event_id) + + hashes = yield self.store.get_event_reference_hashes( + auth_events + ) + hashes = [ + { + k: encode_base64(v) for k, v in h.items() + if k == "sha256" + } + for h in hashes + ] + event.auth_events = zip(auth_events, hashes) + @defer.inlineCallbacks def _on_new_room_event(self, event, snapshot, extra_destinations=[], extra_users=[], suppress_auth=False): @@ -64,6 +115,8 @@ class BaseHandler(object): yield self.state_handler.annotate_state_groups(event) + yield self._add_auth(event) + logger.debug("Signing event...") add_hashes_and_signatures( @@ -76,6 +129,8 @@ class BaseHandler(object): logger.debug("Authing...") self.auth.check(event, raises=True) logger.debug("Authed") + else: + logger.debug("Suppressed auth.") yield self.store.persist_event(event) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 2a1970914f..48ad4d864f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -19,7 +19,6 @@ from synapse.api.events.room import ( RoomMemberEvent, RoomTopicEvent, FeedbackEvent, RoomNameEvent, RoomJoinRulesEvent, - RoomPowerLevelsEvent, RoomRedactionEvent, ) @@ -302,6 +301,17 @@ class DataStore(RoomMemberStore, RoomStore, txn, event.event_id, prev_event_id, alg, hash_bytes ) + for auth_id, _ in event.auth_events: + self._simple_insert_txn( + txn, + table="event_auth", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "auth_id": auth_id, + }, + ) + (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) self._store_event_reference_hash_txn( txn, event.event_id, ref_alg, ref_hash_bytes diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 7821fc4726..9aa404695d 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -474,6 +474,8 @@ class SQLBaseStore(object): if is_state == 0 ] + ev.auth_events = self._get_auth_events(txn, ev.event_id) + if hasattr(ev, "state_key"): ev.prev_state = [ (e_id, h) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 180a764134..86c68ebf87 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -139,6 +139,27 @@ class EventFederationStore(SQLBaseStore): return results + def _get_auth_events(self, txn, event_id): + auth_ids = self._simple_select_onecol_txn( + txn, + table="event_auth", + keyvalues={ + "event_id": event_id, + }, + retcol="auth_id", + ) + + results = [] + for auth_id in auth_ids: + hashes = self._get_event_reference_hashes_txn(txn, auth_id) + prev_hashes = { + k: encode_base64(v) for k, v in hashes.items() + if k == "sha256" + } + results.append((auth_id, prev_hashes)) + + return results + def get_min_depth(self, room_id): return self.runInteraction( "get_min_depth", diff --git a/synapse/storage/schema/event_edges.sql b/synapse/storage/schema/event_edges.sql index 51695826a8..be1c72a775 100644 --- a/synapse/storage/schema/event_edges.sql +++ b/synapse/storage/schema/event_edges.sql @@ -63,3 +63,13 @@ CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( ); CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); + +CREATE TABLE IF NOT EXISTS event_auth( + event_id TEXT NOT NULL, + auth_id TEXT NOT NULL, + room_id TEXT NOT NULL, + CONSTRAINT uniqueness UNIQUE (event_id, auth_id, room_id) +); + +CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); +CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); \ No newline at end of file diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index b4b3d5d7ea..84a49088a2 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -55,6 +55,18 @@ class SignatureStore(SQLBaseStore): or_ignore=True, ) + def get_event_reference_hashes(self, event_ids): + def f(txn): + return [ + self._get_event_reference_hashes_txn(txn, ev) + for ev in event_ids + ] + + return self.runInteraction( + "get_event_reference_hashes", + f + ) + def _get_event_reference_hashes_txn(self, txn, event_id): """Get all the hashes for a given PDU. Args: From 8b0e96474b3a18866f28f112de7e56979b401768 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 10:42:31 +0000 Subject: [PATCH 091/227] Implement method to get auth_chain from a given event_id --- synapse/storage/event_federation.py | 35 +++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 86c68ebf87..7140ea3d57 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -24,6 +24,41 @@ logger = logging.getLogger(__name__) class EventFederationStore(SQLBaseStore): + def get_auth_chain(self, event_id): + return self.runInteraction( + "get_auth_chain", + self._get_auth_chain_txn, + event_id + ) + + def _get_auth_chain_txn(self, txn, event_id): + results = set([event_id]) + + front = set([event_id]) + while front: + for ev_id in front: + new_front = set() + auth_ids = self._simple_select_onecol_txn( + txn, + table="event_auth", + keyvalues={ + "event_id": ev_id, + }, + retcol="auth_id", + ) + + new_front.update(auth_ids) + front = new_front + new_front.clear() + + sql = "SELECT * FROM events WHERE event_id = ?" + rows = [] + for ev_id in results: + c = txn.execute(sql, (ev_id,)) + rows.extend(self.cursor_to_dict(c)) + + return self._parse_events_txn(txn, rows) + def get_oldest_events_in_room(self, room_id): return self.runInteraction( "get_oldest_events_in_room", From 49948d72f3c76f0ca32b844955d3add7922180ea Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 10:42:44 +0000 Subject: [PATCH 092/227] Fix joining over federation --- synapse/api/auth.py | 53 +++++++++++++++++++++++++++++++-- synapse/api/events/__init__.py | 2 +- synapse/handlers/_base.py | 54 +--------------------------------- synapse/handlers/federation.py | 1 + synapse/storage/__init__.py | 1 + 5 files changed, 55 insertions(+), 56 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index e1302553d7..d4f284bd60 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -24,6 +24,7 @@ from synapse.api.events.room import ( RoomJoinRulesEvent, RoomCreateEvent, ) from synapse.util.logutils import log_function +from syutil.base64util import encode_base64 import logging @@ -61,8 +62,6 @@ class Auth(object): # FIXME return True - self._can_send_event(event) - if event.type == RoomMemberEvent.TYPE: allowed = self.is_membership_change_allowed(event) if allowed: @@ -71,6 +70,8 @@ class Auth(object): logger.debug("Denying! %s", event) return allowed + self._can_send_event(event) + if event.type == RoomPowerLevelsEvent.TYPE: self._check_power_levels(event) @@ -311,6 +312,54 @@ class Auth(object): def is_server_admin(self, user): return self.store.is_server_admin(user) + @defer.inlineCallbacks + def add_auth_events(self, event): + if event.type == RoomCreateEvent.TYPE: + event.auth_events = [] + return + + auth_events = [] + + key = (RoomPowerLevelsEvent.TYPE, "", ) + power_level_event = event.old_state_events.get(key) + + if power_level_event: + auth_events.append(power_level_event.event_id) + + key = (RoomJoinRulesEvent.TYPE, "", ) + join_rule_event = event.old_state_events.get(key) + + key = (RoomMemberEvent.TYPE, event.user_id, ) + member_event = event.old_state_events.get(key) + + if join_rule_event: + join_rule = join_rule_event.content.get("join_rule") + is_public = join_rule == JoinRules.PUBLIC if join_rule else False + + if event.type == RoomMemberEvent.TYPE: + if event.content["membership"] == Membership.JOIN: + if is_public: + auth_events.append(join_rule_event.event_id) + elif member_event: + auth_events.append(member_event.event_id) + + if member_event: + if member_event.content["membership"] == Membership.JOIN: + auth_events.append(member_event.event_id) + + hashes = yield self.store.get_event_reference_hashes( + auth_events + ) + hashes = [ + { + k: encode_base64(v) for k, v in h.items() + if k == "sha256" + } + for h in hashes + ] + event.auth_events = zip(auth_events, hashes) + + @log_function def _can_send_event(self, event): key = (RoomPowerLevelsEvent.TYPE, "", ) diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 513a48f568..e5980c4be3 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -61,7 +61,6 @@ class SynapseEvent(JsonEncodedObject): "replaces_state", "redacted_because", "origin_server_ts", - "auth_events", ] internal_keys = [ @@ -75,6 +74,7 @@ class SynapseEvent(JsonEncodedObject): "hashes", "signatures", "prev_state", + "auth_events", ] required_keys = [ diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 2613fa7fce..f630280031 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -18,11 +18,6 @@ from twisted.internet import defer from synapse.api.errors import LimitExceededError from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.api.events.room import ( - RoomCreateEvent, RoomMemberEvent, RoomPowerLevelsEvent, RoomJoinRulesEvent, -) -from synapse.api.constants import Membership, JoinRules -from syutil.base64util import encode_base64 import logging @@ -59,53 +54,6 @@ class BaseHandler(object): retry_after_ms=int(1000*(time_allowed - time_now)), ) - @defer.inlineCallbacks - def _add_auth(self, event): - if event.type == RoomCreateEvent.TYPE: - event.auth_events = [] - return - - auth_events = [] - - key = (RoomPowerLevelsEvent.TYPE, "", ) - power_level_event = event.old_state_events.get(key) - - if power_level_event: - auth_events.append(power_level_event.event_id) - - key = (RoomJoinRulesEvent.TYPE, "", ) - join_rule_event = event.old_state_events.get(key) - - key = (RoomMemberEvent.TYPE, event.user_id, ) - member_event = event.old_state_events.get(key) - - if join_rule_event: - join_rule = join_rule_event.content.get("join_rule") - is_public = join_rule == JoinRules.PUBLIC if join_rule else False - - if event.type == RoomMemberEvent.TYPE: - if event.content["membership"] == Membership.JOIN: - if is_public: - auth_events.append(join_rule_event.event_id) - elif member_event: - auth_events.append(member_event.event_id) - - if member_event: - if member_event.content["membership"] == Membership.JOIN: - auth_events.append(member_event.event_id) - - hashes = yield self.store.get_event_reference_hashes( - auth_events - ) - hashes = [ - { - k: encode_base64(v) for k, v in h.items() - if k == "sha256" - } - for h in hashes - ] - event.auth_events = zip(auth_events, hashes) - @defer.inlineCallbacks def _on_new_room_event(self, event, snapshot, extra_destinations=[], extra_users=[], suppress_auth=False): @@ -115,7 +63,7 @@ class BaseHandler(object): yield self.state_handler.annotate_state_groups(event) - yield self._add_auth(event) + yield self.auth.add_auth_events(event) logger.debug("Signing event...") diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index f0448a05d8..09593303a4 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -317,6 +317,7 @@ class FederationHandler(BaseHandler): snapshot.fill_out_prev_events(event) yield self.state_handler.annotate_state_groups(event) + yield self.auth.add_auth_events(event) self.auth.check(event, raises=True) pdu = self.pdu_codec.pdu_from_event(event) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 48ad4d864f..96adf20c89 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -310,6 +310,7 @@ class DataStore(RoomMemberStore, RoomStore, "room_id": event.room_id, "auth_id": auth_id, }, + or_ignore=True, ) (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) From 16a0815fac750863588c3c1f72c5e445d14bbf43 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 11:21:20 +0000 Subject: [PATCH 093/227] Fix bug in _get_auth_chain_txn --- synapse/storage/event_federation.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 7140ea3d57..d66a49e9f2 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -32,24 +32,21 @@ class EventFederationStore(SQLBaseStore): ) def _get_auth_chain_txn(self, txn, event_id): - results = set([event_id]) + results = set() + + base_sql = ( + "SELECT auth_id FROM event_auth WHERE %s" + ) front = set([event_id]) while front: - for ev_id in front: - new_front = set() - auth_ids = self._simple_select_onecol_txn( - txn, - table="event_auth", - keyvalues={ - "event_id": ev_id, - }, - retcol="auth_id", - ) + sql = base_sql % ( + " OR ".join(["event_id=?"] * len(front)), + ) - new_front.update(auth_ids) - front = new_front - new_front.clear() + txn.execute(sql, list(front)) + front = [r[0] for r in txn.fetchall()] + results.update(front) sql = "SELECT * FROM events WHERE event_id = ?" rows = [] From 3b4dec442da51c6c999dd946db6ea6ce5f07ff0c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 11:22:12 +0000 Subject: [PATCH 094/227] Return auth chain when handling send_join --- synapse/federation/replication.py | 20 +++++++++++++++----- synapse/handlers/federation.py | 15 ++++++++++++--- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 92a9678e2c..d1eddf249d 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -426,8 +426,12 @@ class ReplicationLayer(object): @defer.inlineCallbacks def on_send_join_request(self, origin, content): pdu = Pdu(**content) - state = yield self.handler.on_send_join_request(origin, pdu) - defer.returnValue((200, self._transaction_from_pdus(state).get_dict())) + res_pdus = yield self.handler.on_send_join_request(origin, pdu) + + defer.returnValue((200, { + "state": [p.get_dict() for p in res_pdus["state"]], + "auth_chain": [p.get_dict() for p in res_pdus["auth_chain"]], + })) @defer.inlineCallbacks def make_join(self, destination, context, user_id): @@ -451,11 +455,17 @@ class ReplicationLayer(object): ) logger.debug("Got content: %s", content) - pdus = [Pdu(outlier=True, **p) for p in content.get("pdus", [])] - for pdu in pdus: + state = [Pdu(outlier=True, **p) for p in content.get("state", [])] + for pdu in state: yield self._handle_new_pdu(destination, pdu) - defer.returnValue(pdus) + auth_chain = [ + Pdu(outlier=True, **p) for p in content.get("auth_chain", []) + ] + for pdu in auth_chain: + yield self._handle_new_pdu(destination, pdu) + + defer.returnValue(state) @log_function def _get_persisted_pdu(self, event_id): diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 09593303a4..c193da12b7 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -366,10 +366,19 @@ class FederationHandler(BaseHandler): yield self.replication_layer.send_pdu(new_pdu) - defer.returnValue([ + auth_chain = yield self.store.get_auth_chain(event.event_id) + pdu_auth_chain = [ self.pdu_codec.pdu_from_event(e) - for e in event.state_events.values() - ]) + for e in auth_chain + ] + + defer.returnValue({ + "state": [ + self.pdu_codec.pdu_from_event(e) + for e in event.state_events.values() + ], + "auth_chain": pdu_auth_chain, + }) @defer.inlineCallbacks def get_state_for_pdu(self, event_id): From 97a096b507b92d70a2e07d49122b4f5d93b7dac4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 11:36:40 +0000 Subject: [PATCH 095/227] Add hash of current state to events --- synapse/api/events/__init__.py | 1 + synapse/crypto/event_signing.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index e5980c4be3..8d65c29ac1 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -75,6 +75,7 @@ class SynapseEvent(JsonEncodedObject): "signatures", "prev_state", "auth_events", + "state_hash", ] required_keys = [ diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index de5d2e7465..7d800615fe 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -45,7 +45,7 @@ def check_event_content_hash(event, hash_algorithm=hashlib.sha256): def _compute_content_hash(event, hash_algorithm): event_json = event.get_full_dict() - #TODO: We need to sign the JSON that is going out via fedaration. + # TODO: We need to sign the JSON that is going out via fedaration. event_json.pop("age_ts", None) event_json.pop("unsigned", None) event_json.pop("signatures", None) @@ -81,6 +81,15 @@ def compute_event_signature(event, signature_name, signing_key): def add_hashes_and_signatures(event, signature_name, signing_key, hash_algorithm=hashlib.sha256): + if hasattr(event, "old_state_events"): + state_json_bytes = encode_canonical_json( + [e.event_id for e in event.old_state_events.values()] + ) + hashed = hash_algorithm(state_json_bytes) + event.state_hash = { + hashed.name: encode_base64(hashed.digest()) + } + hashed = _compute_content_hash(event, hash_algorithm=hash_algorithm) if not hasattr(event, "hashes"): From 328dab246376814c141095d63e5cef3b4cb52068 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 11:40:38 +0000 Subject: [PATCH 096/227] Remove /context/ request --- synapse/federation/replication.py | 7 ------- synapse/federation/transport.py | 9 --------- 2 files changed, 16 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index d1eddf249d..37e7db0536 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -272,13 +272,6 @@ class ReplicationLayer(object): defer.returnValue(pdus) - @defer.inlineCallbacks - @log_function - def on_context_pdus_request(self, context): - raise NotImplementedError( - "on_context_pdus_request is a security violation" - ) - @defer.inlineCallbacks @log_function def on_backfill_request(self, context, versions, limit): diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 04ad7e63ae..b9f7d54c71 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -403,15 +403,6 @@ class TransportLayer(object): ) ) - self.server.register_path( - "GET", - re.compile("^" + PREFIX + "/context/([^/]*)/$"), - self._with_authentication( - lambda origin, content, query, context: - handler.on_context_pdus_request(context) - ) - ) - # This is when we receive a server-server Query self.server.register_path( "GET", From d2fb2b8095ec7f5d00b51418e84b05a1b23b79b3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 13:41:00 +0000 Subject: [PATCH 097/227] Implement invite part of invite join dance --- synapse/federation/replication.py | 15 ++++++++++++- synapse/handlers/_base.py | 13 ++++++++++- synapse/handlers/federation.py | 37 +++++++++++++++++++++++++++++++ synapse/handlers/room.py | 30 +++++++++---------------- 4 files changed, 74 insertions(+), 21 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 37e7db0536..e358de942e 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -413,7 +413,7 @@ class ReplicationLayer(object): @defer.inlineCallbacks def on_invite_request(self, origin, content): pdu = Pdu(**content) - ret_pdu = yield self.handler.on_send_join_request(origin, pdu) + ret_pdu = yield self.handler.on_invite_request(origin, pdu) defer.returnValue((200, ret_pdu.get_dict())) @defer.inlineCallbacks @@ -460,6 +460,19 @@ class ReplicationLayer(object): defer.returnValue(state) + @defer.inlineCallbacks + def send_invite(self, destination, context, event_id, pdu): + code, pdu_dict = yield self.transport_layer.send_invite( + destination=destination, + context=context, + event_id=event_id, + content=pdu.get_dict(), + ) + + logger.debug("Got response to send_invite: %s", pdu_dict) + + defer.returnValue(Pdu(**pdu_dict)) + @log_function def _get_persisted_pdu(self, event_id): """ Get a PDU from the database with given origin and id. diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index f630280031..9dc2fc2e0f 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -56,7 +56,8 @@ class BaseHandler(object): @defer.inlineCallbacks def _on_new_room_event(self, event, snapshot, extra_destinations=[], - extra_users=[], suppress_auth=False): + extra_users=[], suppress_auth=False, + do_invite_host=None): yield run_on_reactor() snapshot.fill_out_prev_events(event) @@ -80,6 +81,16 @@ class BaseHandler(object): else: logger.debug("Suppressed auth.") + if do_invite_host: + federation_handler = self.hs.get_handlers().federation_handler + invite_event = yield federation_handler.send_invite( + do_invite_host, + event + ) + + # FIXME: We need to check if the remote changed anything else + event.signatures = invite_event.signatures + yield self.store.persist_event(event) destinations = set(extra_destinations) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c193da12b7..e6afd95a58 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -23,6 +23,7 @@ from synapse.api.constants import Membership from synapse.util.logutils import log_function from synapse.federation.pdu_codec import PduCodec from synapse.util.async import run_on_reactor +from synapse.crypto.event_signing import compute_event_signature from twisted.internet import defer @@ -212,6 +213,17 @@ class FederationHandler(BaseHandler): defer.returnValue(events) + @defer.inlineCallbacks + def send_invite(self, target_host, event): + pdu = yield self.replication_layer.send_invite( + destination=target_host, + context=event.room_id, + event_id=event.event_id, + pdu=self.pdu_codec.pdu_from_event(event) + ) + + defer.returnValue(self.pdu_codec.event_from_pdu(pdu)) + @log_function @defer.inlineCallbacks def do_invite_join(self, target_host, room_id, joinee, content, snapshot): @@ -380,6 +392,31 @@ class FederationHandler(BaseHandler): "auth_chain": pdu_auth_chain, }) + @defer.inlineCallbacks + def on_invite_request(self, origin, pdu): + event = self.pdu_codec.event_from_pdu(pdu) + + event.outlier = True + + event.signatures.update( + compute_event_signature( + event, + self.hs.hostname, + self.hs.config.signing_key[0] + ) + ) + + yield self.state_handler.annotate_state_groups(event) + + yield self.store.persist_event( + event, + backfilled=False, + ) + + yield self.notifier.on_new_room_event(event) + + defer.returnValue(self.pdu_codec.pdu_from_event(event)) + @defer.inlineCallbacks def get_state_for_pdu(self, event_id): yield run_on_reactor() diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 42a6c9f9bf..3642fcfc6d 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -361,13 +361,6 @@ class RoomMemberHandler(BaseHandler): if prev_state: event.content["prev"] = prev_state.membership -# if prev_state and prev_state.membership == event.membership: -# # treat this event as a NOOP. -# if do_auth: # This is mainly to fix a unit test. -# yield self.auth.check(event, raises=True) -# defer.returnValue({}) -# return - room_id = event.room_id # If we're trying to join a room then we have to do this differently @@ -521,25 +514,24 @@ class RoomMemberHandler(BaseHandler): defer.returnValue([r.room_id for r in rooms]) + @defer.inlineCallbacks def _do_local_membership_update(self, event, membership, snapshot, do_auth): - destinations = [] - # If we're inviting someone, then we should also send it to that # HS. target_user_id = event.state_key target_user = self.hs.parse_userid(target_user_id) - if membership == Membership.INVITE: - host = target_user.domain - destinations.append(host) + if membership == Membership.INVITE and not target_user.is_mine: + do_invite_host = target_user.domain + else: + do_invite_host = None - # Always include target domain - host = target_user.domain - destinations.append(host) - - return self._on_new_room_event( - event, snapshot, extra_destinations=destinations, - extra_users=[target_user], suppress_auth=(not do_auth), + yield self._on_new_room_event( + event, + snapshot, + extra_users=[target_user], + suppress_auth=(not do_auth), + do_invite_host=do_invite_host, ) From 02c3b1c9e2afa27753e9ce898e5455b6489541b4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 15:35:53 +0000 Subject: [PATCH 098/227] Add '/event_auth/' federation api --- synapse/federation/replication.py | 5 +++++ synapse/federation/transport.py | 26 ++++++++++++++++++++++++++ synapse/handlers/federation.py | 5 +++++ synapse/storage/event_federation.py | 26 +++++++++++++++++++------- 4 files changed, 55 insertions(+), 7 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index e358de942e..719bfcc42c 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -426,6 +426,11 @@ class ReplicationLayer(object): "auth_chain": [p.get_dict() for p in res_pdus["auth_chain"]], })) + @defer.inlineCallbacks + def on_event_auth(self, origin, context, event_id): + auth_pdus = yield self.handler.on_event_auth(event_id) + defer.returnValue((200, [a.get_dict() for a in auth_pdus])) + @defer.inlineCallbacks def make_join(self, destination, context, user_id): pdu_dict = yield self.transport_layer.make_join( diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index b9f7d54c71..babe8447eb 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -256,6 +256,21 @@ class TransportLayer(object): defer.returnValue(json.loads(content)) + @defer.inlineCallbacks + @log_function + def get_event_auth(self, destination, context, event_id): + path = PREFIX + "/event_auth/%s/%s" % ( + context, + event_id, + ) + + response = yield self.client.get_json( + destination=destination, + path=path, + ) + + defer.returnValue(response) + @defer.inlineCallbacks def _authenticate_request(self, request): json_request = { @@ -426,6 +441,17 @@ class TransportLayer(object): ) ) + self.server.register_path( + "GET", + re.compile("^" + PREFIX + "/event_auth/([^/]*)/([^/]*)$"), + self._with_authentication( + lambda origin, content, query, context, event_id: + handler.on_event_auth( + origin, context, event_id, + ) + ) + ) + self.server.register_path( "PUT", re.compile("^" + PREFIX + "/send_join/([^/]*)/([^/]*)$"), diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index e6afd95a58..ce65bbcd62 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -224,6 +224,11 @@ class FederationHandler(BaseHandler): defer.returnValue(self.pdu_codec.event_from_pdu(pdu)) + @defer.inlineCallbacks + def on_event_auth(self, event_id): + auth = yield self.store.get_auth_chain(event_id) + defer.returnValue([self.pdu_codec.pdu_from_event(e) for e in auth]) + @log_function @defer.inlineCallbacks def do_invite_join(self, target_host, room_id, joinee, content, snapshot): diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index d66a49e9f2..06e32d592d 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -32,6 +32,24 @@ class EventFederationStore(SQLBaseStore): ) def _get_auth_chain_txn(self, txn, event_id): + results = self._get_auth_chain_ids_txn(txn, event_id) + + sql = "SELECT * FROM events WHERE event_id = ?" + rows = [] + for ev_id in results: + c = txn.execute(sql, (ev_id,)) + rows.extend(self.cursor_to_dict(c)) + + return self._parse_events_txn(txn, rows) + + def get_auth_chain_ids(self, event_id): + return self.runInteraction( + "get_auth_chain_ids", + self._get_auth_chain_ids_txn, + event_id + ) + + def _get_auth_chain_ids_txn(self, txn, event_id): results = set() base_sql = ( @@ -48,13 +66,7 @@ class EventFederationStore(SQLBaseStore): front = [r[0] for r in txn.fetchall()] results.update(front) - sql = "SELECT * FROM events WHERE event_id = ?" - rows = [] - for ev_id in results: - c = txn.execute(sql, (ev_id,)) - rows.extend(self.cursor_to_dict(c)) - - return self._parse_events_txn(txn, rows) + return list(results) def get_oldest_events_in_room(self, room_id): return self.runInteraction( From 07286a73b179c7b3888f7cbf15dcf7c4f5a0c3a3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 16:03:31 +0000 Subject: [PATCH 099/227] Use current state to get room hosts, rather than querying the database --- synapse/handlers/_base.py | 18 +++++++++++++++--- synapse/handlers/federation.py | 21 +++++++++++++++++---- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 9dc2fc2e0f..07a8464107 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -18,6 +18,8 @@ from twisted.internet import defer from synapse.api.errors import LimitExceededError from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import add_hashes_and_signatures +from synapse.api.events.room import RoomMemberEvent +from synapse.api.constants import Membership import logging @@ -95,9 +97,19 @@ class BaseHandler(object): destinations = set(extra_destinations) # Send a PDU to all hosts who have joined the room. - destinations.update((yield self.store.get_joined_hosts_for_room( - event.room_id - ))) + + for k, s in event.state_events.items(): + try: + if k[0] == RoomMemberEvent.TYPE: + if s.content["membership"] == Membership.JOIN: + destinations.add( + self.hs.parse_userid(s.state_key).domain + ) + except: + logger.warn( + "Failed to get destination from event %s", s.event_id + ) + event.destinations = list(destinations) self.notifier.on_new_room_event(event, extra_users=extra_users) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ce65bbcd62..7e10583902 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -376,10 +376,23 @@ class FederationHandler(BaseHandler): "user_joined_room", user=user, room_id=event.room_id ) - new_pdu = self.pdu_codec.pdu_from_event(event); - new_pdu.destinations = yield self.store.get_joined_hosts_for_room( - event.room_id - ) + new_pdu = self.pdu_codec.pdu_from_event(event) + + destinations = set() + + for k, s in event.state_events.items(): + try: + if k[0] == RoomMemberEvent.TYPE: + if s.content["membership"] == Membership.JOIN: + destinations.add( + self.hs.parse_userid(s.state_key).domain + ) + except: + logger.warn( + "Failed to get destination from event %s", s.event_id + ) + + new_pdu.destinations = list(destinations) yield self.replication_layer.send_pdu(new_pdu) From 1c06806f90a6368cdc3b9fa3b9053021b7c40e94 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 10:21:32 +0000 Subject: [PATCH 100/227] Finish redaction algorithm. --- synapse/api/events/__init__.py | 4 ++-- synapse/api/events/utils.py | 39 ++++++++++++++++++++++----------- synapse/crypto/event_signing.py | 7 ++---- synapse/federation/units.py | 6 ++--- synapse/storage/_base.py | 2 +- 5 files changed, 33 insertions(+), 25 deletions(-) diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 8d65c29ac1..f1e53f23ab 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -86,8 +86,8 @@ class SynapseEvent(JsonEncodedObject): def __init__(self, raises=True, **kwargs): super(SynapseEvent, self).__init__(**kwargs) - if "content" in kwargs: - self.check_json(self.content, raises=raises) + # if "content" in kwargs: + # self.check_json(self.content, raises=raises) def get_content_template(self): """ Retrieve the JSON template for this event as a dict. diff --git a/synapse/api/events/utils.py b/synapse/api/events/utils.py index 5fc79105b5..802648f8f7 100644 --- a/synapse/api/events/utils.py +++ b/synapse/api/events/utils.py @@ -18,24 +18,31 @@ from .room import ( RoomAliasesEvent, RoomCreateEvent, ) + def prune_event(event): - """ Prunes the given event of all keys we don't know about or think could - potentially be dodgy. + """ Returns a pruned version of the given event, which removes all keys we + don't know about or think could potentially be dodgy. This is used when we "redact" an event. We want to remove all fields that the user has specified, but we do want to keep necessary information like type, state_key etc. """ - return _prune_event_or_pdu(event.type, event) + event_type = event.type -def prune_pdu(pdu): - """Removes keys that contain unrestricted and non-essential data from a PDU - """ - return _prune_event_or_pdu(pdu.type, pdu) - -def _prune_event_or_pdu(event_type, event): - # Remove all extraneous fields. - event.unrecognized_keys = {} + allowed_keys = [ + "event_id", + "user_id", + "room_id", + "hashes", + "signatures", + "content", + "type", + "state_key", + "depth", + "prev_events", + "prev_state", + "auth_events", + ] new_content = {} @@ -65,6 +72,12 @@ def _prune_event_or_pdu(event_type, event): elif event_type == RoomAliasesEvent.TYPE: add_fields("aliases") - event.content = new_content + allowed_fields = { + k: v + for k, v in event.get_full_dict().items() + if k in allowed_keys + } - return event + allowed_fields["content"] = new_content + + return type(event)(**allowed_fields) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 7d800615fe..056e8f6ca4 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -55,9 +55,7 @@ def _compute_content_hash(event, hash_algorithm): def compute_event_reference_hash(event, hash_algorithm=hashlib.sha256): - # FIXME(erikj): GenericEvent! - tmp_event = GenericEvent(**event.get_full_dict()) - tmp_event = prune_event(tmp_event) + tmp_event = prune_event(event) event_json = tmp_event.get_dict() event_json.pop("signatures", None) event_json.pop("age_ts", None) @@ -68,8 +66,7 @@ def compute_event_reference_hash(event, hash_algorithm=hashlib.sha256): def compute_event_signature(event, signature_name, signing_key): - tmp_event = copy.deepcopy(event) - tmp_event = prune_event(tmp_event) + tmp_event = prune_event(event) redact_json = tmp_event.get_full_dict() redact_json.pop("signatures", None) redact_json.pop("age_ts", None) diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 2070ffe1e2..d98014cac7 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -56,17 +56,15 @@ class Pdu(JsonEncodedObject): "origin_server_ts", "type", "destinations", - "transaction_id", "prev_events", "depth", "content", - "outlier", "hashes", + "user_id", + "auth_events", "signatures", # Below this are keys valid only for State Pdus. "state_key", "prev_state", - "required_power_level", - "user_id", ] internal_keys = [ diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 9aa404695d..3ab81a78d5 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -509,7 +509,7 @@ class SQLBaseStore(object): ) if del_evs: - prune_event(ev) + ev = prune_event(ev) ev.redacted_because = del_evs[0] return events From 6cb6cb9e6908ad9b71ebd63ca535eb6c7c48be86 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 10:31:00 +0000 Subject: [PATCH 101/227] Tidy up some of the unused sql tables --- synapse/crypto/event_signing.py | 2 - synapse/storage/__init__.py | 21 ++-------- synapse/storage/room.py | 27 ------------- synapse/storage/schema/im.sql | 68 +++------------------------------ 4 files changed, 9 insertions(+), 109 deletions(-) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 056e8f6ca4..baa93b0ee4 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -19,9 +19,7 @@ from synapse.api.events.utils import prune_event from syutil.jsonutil import encode_canonical_json from syutil.base64util import encode_base64, decode_base64 from syutil.crypto.jsonsign import sign_json -from synapse.api.events.room import GenericEvent -import copy import hashlib import logging diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 96adf20c89..7d810e6a62 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -16,9 +16,7 @@ from twisted.internet import defer from synapse.api.events.room import ( - RoomMemberEvent, RoomTopicEvent, FeedbackEvent, - RoomNameEvent, - RoomJoinRulesEvent, + RoomMemberEvent, RoomTopicEvent, FeedbackEvent, RoomNameEvent, RoomRedactionEvent, ) @@ -95,8 +93,7 @@ class DataStore(RoomMemberStore, RoomStore, @defer.inlineCallbacks @log_function - def persist_event(self, event=None, backfilled=False, pdu=None, - is_new_state=True): + def persist_event(self, event, backfilled=False, is_new_state=True): stream_ordering = None if backfilled: if not self.min_token_deferred.called: @@ -107,8 +104,7 @@ class DataStore(RoomMemberStore, RoomStore, try: yield self.runInteraction( "persist_event", - self._persist_pdu_event_txn, - pdu=pdu, + self._persist_event_txn, event=event, backfilled=backfilled, stream_ordering=stream_ordering, @@ -139,15 +135,6 @@ class DataStore(RoomMemberStore, RoomStore, event = self._parse_event_from_row(events_dict) defer.returnValue(event) - def _persist_pdu_event_txn(self, txn, pdu=None, event=None, - backfilled=False, stream_ordering=None, - is_new_state=True): - if event is not None: - return self._persist_event_txn( - txn, event, backfilled, stream_ordering, - is_new_state=is_new_state, - ) - @log_function def _persist_event_txn(self, txn, event, backfilled, stream_ordering=None, is_new_state=True): @@ -159,8 +146,6 @@ class DataStore(RoomMemberStore, RoomStore, self._store_room_name_txn(txn, event) elif event.type == RoomTopicEvent.TYPE: self._store_room_topic_txn(txn, event) - elif event.type == RoomJoinRulesEvent.TYPE: - self._store_join_rule(txn, event) elif event.type == RoomRedactionEvent.TYPE: self._store_redaction(txn, event) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 0c83c11ad3..ca70506d28 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -132,22 +132,6 @@ class RoomStore(SQLBaseStore): defer.returnValue(ret) - @defer.inlineCallbacks - def get_room_join_rule(self, room_id): - sql = ( - "SELECT join_rule FROM room_join_rules as r " - "INNER JOIN current_state_events as c " - "ON r.event_id = c.event_id " - "WHERE c.room_id = ? " - ) - - rows = yield self._execute(None, sql, room_id) - - if len(rows) == 1: - defer.returnValue(rows[0][0]) - else: - defer.returnValue(None) - def _store_room_topic_txn(self, txn, event): self._simple_insert_txn( txn, @@ -170,17 +154,6 @@ class RoomStore(SQLBaseStore): } ) - def _store_join_rule(self, txn, event): - self._simple_insert_txn( - txn, - "room_join_rules", - { - "event_id": event.event_id, - "room_id": event.room_id, - "join_rule": event.content["join_rule"], - }, - ) - class RoomsTable(Table): table_name = "rooms" diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 8d6f655993..8ba732a23b 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -85,80 +85,24 @@ CREATE TABLE IF NOT EXISTS topics( topic TEXT NOT NULL ); +CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); +CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); + CREATE TABLE IF NOT EXISTS room_names( event_id TEXT NOT NULL, room_id TEXT NOT NULL, name TEXT NOT NULL ); +CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); +CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); + CREATE TABLE IF NOT EXISTS rooms( room_id TEXT PRIMARY KEY NOT NULL, is_public INTEGER, creator TEXT ); -CREATE TABLE IF NOT EXISTS room_join_rules( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - join_rule TEXT NOT NULL -); -CREATE INDEX IF NOT EXISTS room_join_rules_event_id ON room_join_rules(event_id); -CREATE INDEX IF NOT EXISTS room_join_rules_room_id ON room_join_rules(room_id); - - -CREATE TABLE IF NOT EXISTS room_power_levels( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - user_id TEXT NOT NULL, - level INTEGER NOT NULL -); -CREATE INDEX IF NOT EXISTS room_power_levels_event_id ON room_power_levels(event_id); -CREATE INDEX IF NOT EXISTS room_power_levels_room_id ON room_power_levels(room_id); -CREATE INDEX IF NOT EXISTS room_power_levels_room_user ON room_power_levels(room_id, user_id); - - -CREATE TABLE IF NOT EXISTS room_default_levels( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - level INTEGER NOT NULL -); - -CREATE INDEX IF NOT EXISTS room_default_levels_event_id ON room_default_levels(event_id); -CREATE INDEX IF NOT EXISTS room_default_levels_room_id ON room_default_levels(room_id); - - -CREATE TABLE IF NOT EXISTS room_add_state_levels( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - level INTEGER NOT NULL -); - -CREATE INDEX IF NOT EXISTS room_add_state_levels_event_id ON room_add_state_levels(event_id); -CREATE INDEX IF NOT EXISTS room_add_state_levels_room_id ON room_add_state_levels(room_id); - - -CREATE TABLE IF NOT EXISTS room_send_event_levels( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - level INTEGER NOT NULL -); - -CREATE INDEX IF NOT EXISTS room_send_event_levels_event_id ON room_send_event_levels(event_id); -CREATE INDEX IF NOT EXISTS room_send_event_levels_room_id ON room_send_event_levels(room_id); - - -CREATE TABLE IF NOT EXISTS room_ops_levels( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - ban_level INTEGER, - kick_level INTEGER, - redact_level INTEGER -); - -CREATE INDEX IF NOT EXISTS room_ops_levels_event_id ON room_ops_levels(event_id); -CREATE INDEX IF NOT EXISTS room_ops_levels_room_id ON room_ops_levels(room_id); - - CREATE TABLE IF NOT EXISTS room_hosts( room_id TEXT NOT NULL, host TEXT NOT NULL, From 407d8a5019e8e352261886d6479f45e6144ae5b4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 10:35:43 +0000 Subject: [PATCH 102/227] Fix invite auth --- synapse/api/auth.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index d4f284bd60..077d1ab0bf 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -128,13 +128,14 @@ class Auth(object): key = (RoomMemberEvent.TYPE, event.user_id, ) caller = event.old_state_events.get(key) - caller_in_room = caller and caller.membership == "join" + caller_in_room = caller and caller.membership == Membership.JOIN + caller_invited = caller and caller.membership == Membership.INVITE # get info about the target key = (RoomMemberEvent.TYPE, target_user_id, ) target = event.old_state_events.get(key) - target_in_room = target and target.membership == "join" + target_in_room = target and target.membership == Membership.JOIN membership = event.content["membership"] @@ -162,6 +163,7 @@ class Auth(object): "is_membership_change_allowed: %s", { "caller_in_room": caller_in_room, + "caller_invited": caller_invited, "target_in_room": target_in_room, "membership": membership, "join_rule": join_rule, @@ -189,7 +191,7 @@ class Auth(object): elif join_rule == JoinRules.PUBLIC: pass elif join_rule == JoinRules.INVITE: - if not caller_in_room: + if not caller_in_room and not caller_invited: raise AuthError(403, "You are not invited to this room.") else: # TODO (erikj): may_join list From 65f846ade09feba129a14b2b9714c2ad51d00f0a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 11:15:02 +0000 Subject: [PATCH 103/227] Notify users about invites. --- synapse/api/auth.py | 16 +++++++++------- synapse/handlers/federation.py | 5 ++++- synapse/storage/stream.py | 3 +-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 077d1ab0bf..3e5d878eed 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -337,15 +337,17 @@ class Auth(object): if join_rule_event: join_rule = join_rule_event.content.get("join_rule") is_public = join_rule == JoinRules.PUBLIC if join_rule else False + else: + is_public = False - if event.type == RoomMemberEvent.TYPE: - if event.content["membership"] == Membership.JOIN: - if is_public: - auth_events.append(join_rule_event.event_id) - elif member_event: + if event.type == RoomMemberEvent.TYPE: + e_type = event.content["membership"] + if e_type in [Membership.JOIN, Membership.INVITE]: + auth_events.append(join_rule_event.event_id) + + if member_event and not is_public: auth_events.append(member_event.event_id) - - if member_event: + elif member_event: if member_event.content["membership"] == Membership.JOIN: auth_events.append(member_event.event_id) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 7e10583902..9a59fe94d2 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -431,7 +431,10 @@ class FederationHandler(BaseHandler): backfilled=False, ) - yield self.notifier.on_new_room_event(event) + target_user = self.hs.parse_userid(event.state_key) + yield self.notifier.on_new_room_event( + event, extra_users=[target_user], + ) defer.returnValue(self.pdu_codec.pdu_from_event(event)) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 8f7f61d29d..475e7f20a1 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -177,10 +177,9 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT *, (%(redacted)s) AS redacted FROM events AS e WHERE " - "((room_id IN (%(current)s)) OR " + "(e.outlier = 0 AND (room_id IN (%(current)s)) OR " "(event_id IN (%(invites)s))) " "AND e.stream_ordering > ? AND e.stream_ordering <= ? " - "AND e.outlier = 0 " "ORDER BY stream_ordering ASC LIMIT %(limit)d " ) % { "redacted": del_sql, From 6447db063a0d01135582bdfb3392b419f16a19e7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 11:59:51 +0000 Subject: [PATCH 104/227] Fix backfill to work. Add auth to backfill request --- synapse/api/auth.py | 6 +++++ synapse/federation/replication.py | 36 ++++++++++++++++++++++------- synapse/federation/transport.py | 6 ++--- synapse/handlers/federation.py | 10 ++++---- synapse/storage/_base.py | 12 ++++++++++ synapse/storage/event_federation.py | 4 ++-- 6 files changed, 56 insertions(+), 18 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 3e5d878eed..48f9d460a3 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -104,6 +104,12 @@ class Auth(object): pass defer.returnValue(None) + @defer.inlineCallbacks + def check_host_in_room(self, room_id, host): + joined_hosts = yield self.store.get_joined_hosts_for_room(room_id) + + defer.returnValue(host in joined_hosts) + def check_event_sender_in_room(self, event): key = (RoomMemberEvent.TYPE, event.user_id, ) member_event = event.state_events.get(key) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 719bfcc42c..7837f1c252 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -205,7 +205,7 @@ class ReplicationLayer(object): pdus = [Pdu(outlier=False, **p) for p in transaction.pdus] for pdu in pdus: - yield self._handle_new_pdu(pdu, backfilled=True) + yield self._handle_new_pdu(dest, pdu, backfilled=True) defer.returnValue(pdus) @@ -274,9 +274,9 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def on_backfill_request(self, context, versions, limit): + def on_backfill_request(self, origin, context, versions, limit): pdus = yield self.handler.on_backfill_request( - context, versions, limit + origin, context, versions, limit ) defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) @@ -408,13 +408,22 @@ class ReplicationLayer(object): @defer.inlineCallbacks def on_make_join_request(self, context, user_id): pdu = yield self.handler.on_make_join_request(context, user_id) - defer.returnValue(pdu.get_dict()) + defer.returnValue({ + "event": pdu.get_dict(), + }) @defer.inlineCallbacks def on_invite_request(self, origin, content): pdu = Pdu(**content) ret_pdu = yield self.handler.on_invite_request(origin, pdu) - defer.returnValue((200, ret_pdu.get_dict())) + defer.returnValue( + ( + 200, + { + "event": ret_pdu.get_dict(), + } + ) + ) @defer.inlineCallbacks def on_send_join_request(self, origin, content): @@ -429,16 +438,25 @@ class ReplicationLayer(object): @defer.inlineCallbacks def on_event_auth(self, origin, context, event_id): auth_pdus = yield self.handler.on_event_auth(event_id) - defer.returnValue((200, [a.get_dict() for a in auth_pdus])) + defer.returnValue( + ( + 200, + { + "auth_chain": [a.get_dict() for a in auth_pdus], + } + ) + ) @defer.inlineCallbacks def make_join(self, destination, context, user_id): - pdu_dict = yield self.transport_layer.make_join( + ret = yield self.transport_layer.make_join( destination=destination, context=context, user_id=user_id, ) + pdu_dict = ret["event"] + logger.debug("Got response to make_join: %s", pdu_dict) defer.returnValue(Pdu(**pdu_dict)) @@ -467,13 +485,15 @@ class ReplicationLayer(object): @defer.inlineCallbacks def send_invite(self, destination, context, event_id, pdu): - code, pdu_dict = yield self.transport_layer.send_invite( + code, content = yield self.transport_layer.send_invite( destination=destination, context=context, event_id=event_id, content=pdu.get_dict(), ) + pdu_dict = content["event"] + logger.debug("Got response to send_invite: %s", pdu_dict) defer.returnValue(Pdu(**pdu_dict)) diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index babe8447eb..92a1f4ce17 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -413,7 +413,7 @@ class TransportLayer(object): self._with_authentication( lambda origin, content, query, context: self._on_backfill_request( - context, query["v"], query["limit"] + origin, context, query["v"], query["limit"] ) ) ) @@ -552,7 +552,7 @@ class TransportLayer(object): defer.returnValue(data) @log_function - def _on_backfill_request(self, context, v_list, limits): + def _on_backfill_request(self, origin, context, v_list, limits): if not limits: return defer.succeed( (400, {"error": "Did not include limit param"}) @@ -563,7 +563,7 @@ class TransportLayer(object): versions = v_list return self.request_handler.on_backfill_request( - context, versions, limit + origin, context, versions, limit ) @defer.inlineCallbacks diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 9a59fe94d2..00d10609b8 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -193,10 +193,7 @@ class FederationHandler(BaseHandler): dest, room_id, limit, - extremities=[ - self.pdu_codec.decode_event_id(e) - for e in extremities - ] + extremities=extremities, ) events = [] @@ -473,7 +470,10 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks @log_function - def on_backfill_request(self, context, pdu_list, limit): + def on_backfill_request(self, origin, context, pdu_list, limit): + in_room = yield self.auth.check_host_in_room(context, origin) + if not in_room: + raise AuthError(403, "Host not in room.") events = yield self.store.get_backfill_events( context, diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 3ab81a78d5..a23f2b941b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -447,6 +447,18 @@ class SQLBaseStore(object): **d ) + def _get_events_txn(self, txn, event_ids): + # FIXME (erikj): This should be batched? + + sql = "SELECT * FROM events WHERE event_id = ?" + + event_rows = [] + for e_id in event_ids: + c = txn.execute(sql, (e_id,)) + event_rows.extend(self.cursor_to_dict(c)) + + return self._parse_events_txn(txn, event_rows) + def _parse_events(self, rows): return self.runInteraction( "_parse_events", self._parse_events_txn, rows diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 06e32d592d..a707030145 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -371,10 +371,10 @@ class EventFederationStore(SQLBaseStore): "_backfill_interaction: got id=%s", *row ) - new_front.append(row) + new_front.append(row[0]) front = new_front event_results += new_front # We also want to update the `prev_pdus` attributes before returning. - return self._get_pdu_tuples(txn, event_results) + return self._get_events_txn(txn, event_results) From 003668cfaadc2e96fab0127d7c563eb4b17e0619 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 13:37:24 +0000 Subject: [PATCH 105/227] Add auth to the various server-server APIs --- synapse/federation/replication.py | 14 ++++++++------ synapse/federation/transport.py | 3 ++- synapse/handlers/federation.py | 15 +++++++++++++-- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 7837f1c252..dd8124dbb9 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -347,10 +347,12 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def on_context_state_request(self, context, event_id): + def on_context_state_request(self, origin, context, event_id): if event_id: pdus = yield self.handler.get_state_for_pdu( - event_id + origin, + context, + event_id, ) else: raise NotImplementedError("Specify an event") @@ -365,8 +367,8 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def on_pdu_request(self, event_id): - pdu = yield self._get_persisted_pdu(event_id) + def on_pdu_request(self, origin, event_id): + pdu = yield self._get_persisted_pdu(origin, event_id) if pdu: defer.returnValue( @@ -499,13 +501,13 @@ class ReplicationLayer(object): defer.returnValue(Pdu(**pdu_dict)) @log_function - def _get_persisted_pdu(self, event_id): + def _get_persisted_pdu(self, origin, event_id): """ Get a PDU from the database with given origin and id. Returns: Deferred: Results in a `Pdu`. """ - return self.handler.get_persisted_pdu(event_id) + return self.handler.get_persisted_pdu(origin, event_id) def _transaction_from_pdus(self, pdu_list): """Returns a new Transaction containing the given PDUs suitable for diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 92a1f4ce17..d84a44c211 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -390,7 +390,7 @@ class TransportLayer(object): re.compile("^" + PREFIX + "/event/([^/]*)/$"), self._with_authentication( lambda origin, content, query, event_id: - handler.on_pdu_request(event_id) + handler.on_pdu_request(origin, event_id) ) ) @@ -401,6 +401,7 @@ class TransportLayer(object): self._with_authentication( lambda origin, content, query, context: handler.on_context_state_request( + origin, context, query.get("event_id", [None])[0], ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 00d10609b8..587fa308c8 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -436,9 +436,13 @@ class FederationHandler(BaseHandler): defer.returnValue(self.pdu_codec.pdu_from_event(event)) @defer.inlineCallbacks - def get_state_for_pdu(self, event_id): + def get_state_for_pdu(self, origin, room_id, event_id): yield run_on_reactor() + in_room = yield self.auth.check_host_in_room(room_id, origin) + if not in_room: + raise AuthError(403, "Host not in room.") + state_groups = yield self.store.get_state_groups( [event_id] ) @@ -488,7 +492,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks @log_function - def get_persisted_pdu(self, event_id): + def get_persisted_pdu(self, origin, event_id): """ Get a PDU from the database with given origin and id. Returns: @@ -500,6 +504,13 @@ class FederationHandler(BaseHandler): ) if event: + in_room = yield self.auth.check_host_in_room( + event.room_id, + origin + ) + if not in_room: + raise AuthError(403, "Host not in room.") + defer.returnValue(self.pdu_codec.pdu_from_event(event)) else: defer.returnValue(None) From c46088405aa6baa41da0aef56073ae3b4fe9f850 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 13:39:33 +0000 Subject: [PATCH 106/227] Remove useless comments --- synapse/federation/replication.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index dd8124dbb9..e798304353 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -356,12 +356,6 @@ class ReplicationLayer(object): ) else: raise NotImplementedError("Specify an event") - # results = yield self.store.get_current_state_for_context( - # context - # ) - # pdus = [Pdu.from_pdu_tuple(p) for p in results] - # - # logger.debug("Context returning %d results", len(pdus)) defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) @@ -382,21 +376,6 @@ class ReplicationLayer(object): def on_pull_request(self, origin, versions): raise NotImplementedError("Pull transacions not implemented") - # transaction_id = max([int(v) for v in versions]) - # - # response = yield self.pdu_actions.after_transaction( - # transaction_id, - # origin, - # self.server_name - # ) - # - # if not response: - # response = [] - # - # defer.returnValue( - # (200, self._transaction_from_pdus(response).get_dict()) - # ) - @defer.inlineCallbacks def on_query_request(self, query_type, args): if query_type in self.query_handlers: @@ -570,8 +549,6 @@ class ReplicationLayer(object): origin, pdu.room_id, pdu.event_id, ) - # Persist the Pdu, but don't mark it as processed yet. - # yield self.store.persist_event(pdu=pdu) if not backfilled: ret = yield self.handler.on_receive_pdu( From 5d439b127ba34b951dfd09a7d3c684c2d50df702 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 13:46:44 +0000 Subject: [PATCH 107/227] PEP8 --- synapse/api/auth.py | 3 +-- synapse/api/events/room.py | 1 + synapse/federation/replication.py | 1 - synapse/federation/transport.py | 9 ++++++--- synapse/federation/units.py | 7 +++---- synapse/handlers/federation.py | 5 ++++- synapse/storage/__init__.py | 7 ++++--- synapse/storage/event_federation.py | 9 +++------ 8 files changed, 22 insertions(+), 20 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 48f9d460a3..a5c6964707 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -369,7 +369,6 @@ class Auth(object): ] event.auth_events = zip(auth_events, hashes) - @log_function def _can_send_event(self, event): key = (RoomPowerLevelsEvent.TYPE, "", ) @@ -452,7 +451,7 @@ class Auth(object): event.user_id, ) - # Check other levels: + # Check other levels: levels_to_check = [ ("users_default", []), ("events_default", []), diff --git a/synapse/api/events/room.py b/synapse/api/events/room.py index 25bc883706..8c4ac45d02 100644 --- a/synapse/api/events/room.py +++ b/synapse/api/events/room.py @@ -153,6 +153,7 @@ class RoomPowerLevelsEvent(SynapseStateEvent): def get_content_template(self): return {} + class RoomAliasesEvent(SynapseStateEvent): TYPE = "m.room.aliases" diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index e798304353..bacba36755 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -549,7 +549,6 @@ class ReplicationLayer(object): origin, pdu.room_id, pdu.event_id, ) - if not backfilled: ret = yield self.handler.on_receive_pdu( pdu, diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index d84a44c211..95c40c6c1b 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -284,7 +284,7 @@ class TransportLayer(object): origin = None if request.method == "PUT": - #TODO: Handle other method types? other content types? + # TODO: Handle other method types? other content types? try: content_bytes = request.content.read() content = json.loads(content_bytes) @@ -296,11 +296,13 @@ class TransportLayer(object): try: params = auth.split(" ")[1].split(",") param_dict = dict(kv.split("=") for kv in params) + def strip_quotes(value): if value.startswith("\""): return value[1:-1] else: return value + origin = strip_quotes(param_dict["origin"]) key = strip_quotes(param_dict["key"]) sig = strip_quotes(param_dict["sig"]) @@ -321,7 +323,7 @@ class TransportLayer(object): if auth.startswith("X-Matrix"): (origin, key, sig) = parse_auth_header(auth) json_request["origin"] = origin - json_request["signatures"].setdefault(origin,{})[key] = sig + json_request["signatures"].setdefault(origin, {})[key] = sig if not json_request["signatures"]: raise SynapseError( @@ -515,7 +517,8 @@ class TransportLayer(object): return try: - code, response = yield self.received_handler.on_incoming_transaction( + handler = self.received_handler + code, response = yield handler.on_incoming_transaction( transaction_data ) except: diff --git a/synapse/federation/units.py b/synapse/federation/units.py index d98014cac7..f4e7b62bd9 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -192,7 +192,9 @@ class Transaction(JsonEncodedObject): transaction_id and origin_server_ts keys. """ if "origin_server_ts" not in kwargs: - raise KeyError("Require 'origin_server_ts' to construct a Transaction") + raise KeyError( + "Require 'origin_server_ts' to construct a Transaction" + ) if "transaction_id" not in kwargs: raise KeyError( "Require 'transaction_id' to construct a Transaction" @@ -204,6 +206,3 @@ class Transaction(JsonEncodedObject): kwargs["pdus"] = [p.get_dict() for p in pdus] return Transaction(**kwargs) - - - diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 587fa308c8..e909af6bd8 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -521,6 +521,9 @@ class FederationHandler(BaseHandler): @log_function def _on_user_joined(self, user, room_id): - waiters = self.waiting_for_join_list.get((user.to_string(), room_id), []) + waiters = self.waiting_for_join_list.get( + (user.to_string(), room_id), + [] + ) while waiters: waiters.pop().callback(None) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 7d810e6a62..4034437f6b 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -494,11 +494,13 @@ def prepare_database(db_conn): user_version = row[0] if user_version > SCHEMA_VERSION: - raise ValueError("Cannot use this database as it is too " + + raise ValueError( + "Cannot use this database as it is too " + "new for the server to understand" ) elif user_version < SCHEMA_VERSION: - logging.info("Upgrading database from version %d", + logging.info( + "Upgrading database from version %d", user_version ) @@ -520,4 +522,3 @@ def prepare_database(db_conn): c.execute("PRAGMA user_version = %d" % SCHEMA_VERSION) c.close() - diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index a707030145..a027db3868 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -215,7 +215,7 @@ class EventFederationStore(SQLBaseStore): min_depth = self._simple_select_one_onecol_txn( txn, table="room_depth", - keyvalues={"room_id": room_id,}, + keyvalues={"room_id": room_id}, retcol="min_depth", allow_none=True, ) @@ -267,10 +267,8 @@ class EventFederationStore(SQLBaseStore): } ) - - - # We only insert as a forward extremity the new pdu if there are no - # other pdus that reference it as a prev pdu + # We only insert as a forward extremity the new pdu if there are + # no other pdus that reference it as a prev pdu query = ( "INSERT OR IGNORE INTO %(table)s (event_id, room_id) " "SELECT ?, ? WHERE NOT EXISTS (" @@ -312,7 +310,6 @@ class EventFederationStore(SQLBaseStore): ) txn.execute(query) - def get_backfill_events(self, room_id, event_list, limit): """Get a list of Events for a given topic that occured before (and including) the pdus in pdu_list. Return a list of max size `limit`. From b01159f2348de61a085d7ea20cc0fc90db2fc080 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 14:58:33 +0000 Subject: [PATCH 108/227] Fix room handler test --- tests/handlers/test_room.py | 55 +++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 146ed21b65..55c9f6e142 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -58,9 +58,8 @@ class RoomMemberHandlerTestCase(unittest.TestCase): "profile_handler", "federation_handler", ]), - auth=NonCallableMock(spec_set=["check"]), + auth=NonCallableMock(spec_set=["check", "add_auth_events"]), state_handler=NonCallableMock(spec_set=[ - "handle_new_event", "annotate_state_groups", ]), config=self.mock_config, @@ -68,6 +67,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.federation = NonCallableMock(spec_set=[ "handle_new_event", + "send_invite", "get_state_for_room", ]) @@ -110,7 +110,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase): joined = ["red", "green"] - self.state_handler.handle_new_event.return_value = defer.succeed(True) self.datastore.get_joined_hosts_for_room.return_value = ( defer.succeed(joined) ) @@ -120,6 +119,18 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.datastore.get_room_member.return_value = defer.succeed(None) + event.state_events = { + (RoomMemberEvent.TYPE, "@alice:green"): self._create_member( + user_id="@alice:green", + room_id=room_id, + ), + (RoomMemberEvent.TYPE, "@bob:red"): self._create_member( + user_id="@bob:red", + room_id=room_id, + ), + (RoomMemberEvent.TYPE, target_user_id): event, + } + # Actual invocation yield self.room_member_handler.change_membership(event) @@ -128,7 +139,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): ) self.assertEquals( - set(["blue", "red", "green"]), + set(["red", "green"]), set(event.destinations) ) @@ -147,22 +158,14 @@ class RoomMemberHandlerTestCase(unittest.TestCase): room_id = "!foo:red" user_id = "@bob:red" user = self.hs.parse_userid(user_id) - target_user_id = "@bob:red" - content = {"membership": Membership.JOIN} - event = self.hs.get_event_factory().create_event( - etype=RoomMemberEvent.TYPE, + event = self._create_member( user_id=user_id, - state_key=target_user_id, room_id=room_id, - membership=Membership.JOIN, - content=content, ) joined = ["red", "green"] - self.state_handler.handle_new_event.return_value = defer.succeed(True) - def get_joined(*args): return defer.succeed(joined) @@ -180,6 +183,14 @@ class RoomMemberHandlerTestCase(unittest.TestCase): join_signal_observer = Mock() self.distributor.observe("user_joined_room", join_signal_observer) + event.state_events = { + (RoomMemberEvent.TYPE, "@alice:green"): self._create_member( + user_id="@alice:green", + room_id=room_id, + ), + (RoomMemberEvent.TYPE, user_id): event, + } + # Actual invocation yield self.room_member_handler.change_membership(event) @@ -203,6 +214,16 @@ class RoomMemberHandlerTestCase(unittest.TestCase): user=user, room_id=room_id ) + def _create_member(self, user_id, room_id): + return self.hs.get_event_factory().create_event( + etype=RoomMemberEvent.TYPE, + user_id=user_id, + state_key=user_id, + room_id=room_id, + membership=Membership.JOIN, + content={"membership": Membership.JOIN}, + ) + class RoomCreationTest(unittest.TestCase): @@ -228,9 +249,8 @@ class RoomCreationTest(unittest.TestCase): "room_member_handler", "federation_handler", ]), - auth=NonCallableMock(spec_set=["check"]), + auth=NonCallableMock(spec_set=["check", "add_auth_events"]), state_handler=NonCallableMock(spec_set=[ - "handle_new_event", "annotate_state_groups", ]), ratelimiter=NonCallableMock(spec_set=[ @@ -259,6 +279,11 @@ class RoomCreationTest(unittest.TestCase): ]) self.room_member_handler = self.handlers.room_member_handler + def annotate(event): + event.state_events = {} + return defer.succeed(None) + self.state_handler.annotate_state_groups.side_effect = annotate + def hosts(room): return defer.succeed([]) self.datastore.get_joined_hosts_for_room.side_effect = hosts From cdc1b5d629b49300de848647bd2a80a024fae8f7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 15:21:30 +0000 Subject: [PATCH 109/227] Fix regression where we did not return redacted events. --- synapse/storage/_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a23f2b941b..2df64bdfeb 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -469,7 +469,7 @@ class SQLBaseStore(object): select_event_sql = "SELECT * FROM events WHERE event_id = ?" - for ev in events: + for i, ev in enumerate(events): signatures = self._get_event_origin_signatures_txn( txn, ev.event_id, ) @@ -522,6 +522,7 @@ class SQLBaseStore(object): if del_evs: ev = prune_event(ev) + events[i] = ev ev.redacted_because = del_evs[0] return events From c174d19d1e107c31fe5d2d2b919bbd49a5b86e56 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 15:21:41 +0000 Subject: [PATCH 110/227] Fix redaction storage test --- tests/storage/test_redaction.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index fb175a5c4c..adfe64a980 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -71,6 +71,8 @@ class RedactionTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.prev_state = [] + event.auth_events = [] # Have to create a join event using the eventfactory yield self.store.persist_event( @@ -94,6 +96,7 @@ class RedactionTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.auth_events = [] yield self.store.persist_event( event @@ -115,6 +118,7 @@ class RedactionTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.auth_events = [] yield self.store.persist_event( event From e2cebe26e8ba1d7c2a39594ea1f4be0dee73ace4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 15:24:15 +0000 Subject: [PATCH 111/227] Fix room_member storage test --- tests/storage/test_roommember.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index 9e2b2e31c8..8614e5ca9d 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -64,6 +64,8 @@ class RoomMemberStoreTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.prev_state = {} + event.auth_events = {} yield self.store.persist_event( event From 3cd9c02f71f967918f494e61121a8ecb18e67611 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 15:29:19 +0000 Subject: [PATCH 112/227] Fix stream test. --- tests/storage/test_stream.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index b034fd8f16..5038546aee 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -48,7 +48,7 @@ class StreamStoreTestCase(unittest.TestCase): self.depth = 1 @defer.inlineCallbacks - def inject_room_member(self, room, user, membership, prev_state=None): + def inject_room_member(self, room, user, membership, replaces_state=None): self.depth += 1 event = self.event_factory.create_event( @@ -64,9 +64,12 @@ class StreamStoreTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.prev_state = [] + event.auth_events = [] - if prev_state: - event.prev_state = prev_state + if replaces_state: + event.prev_state = [(replaces_state, "hash")] + event.replaces_state = replaces_state # Have to create a join event using the eventfactory yield self.store.persist_event( @@ -90,6 +93,7 @@ class StreamStoreTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.auth_events = [] # Have to create a join event using the eventfactory yield self.store.persist_event( @@ -216,7 +220,7 @@ class StreamStoreTestCase(unittest.TestCase): event2 = yield self.inject_room_member( self.room1, self.u_alice, Membership.JOIN, - prev_state=event1.event_id, + replaces_state=event1.event_id, ) end = yield self.store.get_room_events_max_id() @@ -233,4 +237,7 @@ class StreamStoreTestCase(unittest.TestCase): event = results[0] - self.assertTrue(hasattr(event, "prev_content"), msg="No prev_content key") + self.assertTrue( + hasattr(event, "prev_content"), + msg="No prev_content key" + ) From 4ebdb1968225243bab86ef5d8d9f8feb274dd9ce Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 15:32:35 +0000 Subject: [PATCH 113/227] Fix SQLBaseStoreTestCase --- tests/storage/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 3ad9a4b0c0..fabd364be9 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -74,7 +74,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_select_one_1col(self): self.mock_txn.rowcount = 1 - self.mock_txn.fetchone.return_value = ("Value",) + self.mock_txn.fetchall.return_value = [("Value",)] value = yield self.datastore._simple_select_one_onecol( table="tablename", From ec824927c1e1af76abad1ea0c5eea1098a5c9cfa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 15:37:53 +0000 Subject: [PATCH 114/227] Fix rest.test_events. Convert to use SQLiteMemoryDbPool --- tests/rest/test_events.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/rest/test_events.py b/tests/rest/test_events.py index 82f5d277e6..4a3234c332 100644 --- a/tests/rest/test_events.py +++ b/tests/rest/test_events.py @@ -25,7 +25,7 @@ import synapse.rest.room from synapse.server import HomeServer -from ..utils import MockHttpResource, MemoryDataStore, MockKey +from ..utils import MockHttpResource, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -113,25 +113,20 @@ class EventStreamPermissionsTestCase(RestTestCase): def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - state_handler = Mock(spec=[ - "handle_new_event", - "annotate_state_groups" - ]) - state_handler.handle_new_event.return_value = True - persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + hs = HomeServer( "test", - db_pool=None, + db_pool=db_pool, http_client=None, replication_layer=Mock(), - state_handler=state_handler, - datastore=MemoryDataStore(), persistence_service=persistence_service, clock=Mock(spec=[ "call_later", From a8e565eca8cbfcedbdfd812c98a6545c2fc31afd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 18:24:43 +0000 Subject: [PATCH 115/227] Add an EventValidator. Fix bugs in auth ++ storage --- synapse/api/auth.py | 16 ++++++--- synapse/api/events/__init__.py | 61 --------------------------------- synapse/handlers/profile.py | 11 +++--- synapse/rest/base.py | 2 ++ synapse/rest/room.py | 13 +++++++ synapse/server.py | 5 +++ synapse/storage/_base.py | 2 +- synapse/storage/registration.py | 6 +++- synapse/storage/room.py | 38 ++++++++++---------- 9 files changed, 64 insertions(+), 90 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index a5c6964707..6c2d3db26e 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -70,6 +70,7 @@ class Auth(object): logger.debug("Denying! %s", event) return allowed + self.check_event_sender_in_room(event) self._can_send_event(event) if event.type == RoomPowerLevelsEvent.TYPE: @@ -83,8 +84,10 @@ class Auth(object): else: raise AuthError(500, "Unknown event: %s" % event) except AuthError as e: - logger.info("Event auth check failed on event %s with msg: %s", - event, e.msg) + logger.info( + "Event auth check failed on event %s with msg: %s", + event, e.msg + ) logger.info("Denying! %s", event) if raises: raise e @@ -277,7 +280,7 @@ class Auth(object): default=[""] )[0] if user and access_token and ip_addr: - self.store.insert_client_ip( + yield self.store.insert_client_ip( user=user, access_token=access_token, device_id=user_info["device_id"], @@ -349,7 +352,8 @@ class Auth(object): if event.type == RoomMemberEvent.TYPE: e_type = event.content["membership"] if e_type in [Membership.JOIN, Membership.INVITE]: - auth_events.append(join_rule_event.event_id) + if join_rule_event: + auth_events.append(join_rule_event.event_id) if member_event and not is_public: auth_events.append(member_event.event_id) @@ -405,7 +409,9 @@ class Auth(object): if user_level < send_level: raise AuthError( - 403, "You don't have permission to post that to the room" + 403, + "You don't have permission to post that to the room. " + + "user_level (%d) < send_level (%d)" % (user_level, send_level) ) return True diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index f1e53f23ab..1d8bed2906 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.api.errors import SynapseError, Codes from synapse.util.jsonobject import JsonEncodedObject @@ -118,66 +117,6 @@ class SynapseEvent(JsonEncodedObject): """ raise NotImplementedError("get_content_template not implemented.") - def check_json(self, content, raises=True): - """Checks the given JSON content abides by the rules of the template. - - Args: - content : A JSON object to check. - raises: True to raise a SynapseError if the check fails. - Returns: - True if the content passes the template. Returns False if the check - fails and raises=False. - Raises: - SynapseError if the check fails and raises=True. - """ - # recursively call to inspect each layer - err_msg = self._check_json(content, self.get_content_template()) - if err_msg: - if raises: - raise SynapseError(400, err_msg, Codes.BAD_JSON) - else: - return False - else: - return True - - def _check_json(self, content, template): - """Check content and template matches. - - If the template is a dict, each key in the dict will be validated with - the content, else it will just compare the types of content and - template. This basic type check is required because this function will - be recursively called and could be called with just strs or ints. - - Args: - content: The content to validate. - template: The validation template. - Returns: - str: An error message if the validation fails, else None. - """ - if type(content) != type(template): - return "Mismatched types: %s" % template - - if type(template) == dict: - for key in template: - if key not in content: - return "Missing %s key" % key - - if type(content[key]) != type(template[key]): - return "Key %s is of the wrong type (got %s, want %s)" % ( - key, type(content[key]), type(template[key])) - - if type(content[key]) == dict: - # we must go deeper - msg = self._check_json(content[key], template[key]) - if msg: - return msg - elif type(content[key]) == list: - # make sure each item type in content matches the template - for entry in content[key]: - msg = self._check_json(entry, template[key][0]) - if msg: - return msg - class SynapseStateEvent(SynapseEvent): diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index e47814483a..834b37f5f3 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -152,10 +152,13 @@ class ProfileHandler(BaseHandler): if not user.is_mine: defer.returnValue(None) - (displayname, avatar_url) = yield defer.gatherResults([ - self.store.get_profile_displayname(user.localpart), - self.store.get_profile_avatar_url(user.localpart), - ]) + (displayname, avatar_url) = yield defer.gatherResults( + [ + self.store.get_profile_displayname(user.localpart), + self.store.get_profile_avatar_url(user.localpart), + ], + consumeErrors=True + ) state["displayname"] = displayname state["avatar_url"] = avatar_url diff --git a/synapse/rest/base.py b/synapse/rest/base.py index dc784c1527..79fc4dfb84 100644 --- a/synapse/rest/base.py +++ b/synapse/rest/base.py @@ -67,6 +67,8 @@ class RestServlet(object): self.auth = hs.get_auth() self.txns = HttpTransactionStore() + self.validator = hs.get_event_validator() + def register(self, http_server): """ Register this servlet with the given HTTP server. """ if hasattr(self, "PATTERN"): diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 5c9c9d3af4..05da0be090 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -154,6 +154,9 @@ class RoomStateEventRestServlet(RestServlet): user_id=user.to_string(), state_key=urllib.unquote(state_key) ) + + self.validator.validate(event) + if event_type == RoomMemberEvent.TYPE: # membership events are special handler = self.handlers.room_member_handler @@ -188,6 +191,8 @@ class RoomSendEventRestServlet(RestServlet): content=content ) + self.validator.validate(event) + msg_handler = self.handlers.message_handler yield msg_handler.send_message(event) @@ -253,6 +258,9 @@ class JoinRoomAliasServlet(RestServlet): user_id=user.to_string(), state_key=user.to_string() ) + + self.validator.validate(event) + handler = self.handlers.room_member_handler yield handler.change_membership(event) defer.returnValue((200, {})) @@ -424,6 +432,9 @@ class RoomMembershipRestServlet(RestServlet): user_id=user.to_string(), state_key=state_key ) + + self.validator.validate(event) + handler = self.handlers.room_member_handler yield handler.change_membership(event) defer.returnValue((200, {})) @@ -461,6 +472,8 @@ class RoomRedactEventRestServlet(RestServlet): redacts=urllib.unquote(event_id), ) + self.validator.validate(event) + msg_handler = self.handlers.message_handler yield msg_handler.send_message(event) diff --git a/synapse/server.py b/synapse/server.py index d770b20b19..da0a44433a 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -22,6 +22,7 @@ from synapse.federation import initialize_http_replication from synapse.api.events import serialize_event from synapse.api.events.factory import EventFactory +from synapse.api.events.validator import EventValidator from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers @@ -80,6 +81,7 @@ class BaseHomeServer(object): 'event_sources', 'ratelimiter', 'keyring', + 'event_validator', ] def __init__(self, hostname, **kwargs): @@ -223,6 +225,9 @@ class HomeServer(BaseHomeServer): def build_keyring(self): return Keyring(self) + def build_event_validator(self): + return EventValidator(self) + def register_servlets(self): """ Register all servlets associated with this HomeServer. """ diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 2df64bdfeb..a1ee0318f6 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -212,7 +212,7 @@ class SQLBaseStore(object): retcol : string giving the name of the column to return """ return self.runInteraction( - "_simple_select_one_onecol_txn", + "_simple_select_one_onecol", self._simple_select_one_onecol_txn, table, keyvalues, retcol, allow_none=allow_none, ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index a2ca6f9a69..1f89d77344 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -107,13 +107,17 @@ class RegistrationStore(SQLBaseStore): token ) + @defer.inlineCallbacks def is_server_admin(self, user): - return self._simple_select_one_onecol( + res = yield self._simple_select_one_onecol( table="users", keyvalues={"name": user.to_string()}, retcol="admin", + allow_none=True, ) + defer.returnValue(res if res else False) + def _query_for_auth(self, txn, token): sql = ( "SELECT users.name, users.admin, access_tokens.device_id " diff --git a/synapse/storage/room.py b/synapse/storage/room.py index ca70506d28..cc0513b8d2 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -133,26 +133,28 @@ class RoomStore(SQLBaseStore): defer.returnValue(ret) def _store_room_topic_txn(self, txn, event): - self._simple_insert_txn( - txn, - "topics", - { - "event_id": event.event_id, - "room_id": event.room_id, - "topic": event.topic, - } - ) + if hasattr(event, "topic"): + self._simple_insert_txn( + txn, + "topics", + { + "event_id": event.event_id, + "room_id": event.room_id, + "topic": event.topic, + } + ) def _store_room_name_txn(self, txn, event): - self._simple_insert_txn( - txn, - "room_names", - { - "event_id": event.event_id, - "room_id": event.room_id, - "name": event.name, - } - ) + if hasattr(event, "name"): + self._simple_insert_txn( + txn, + "room_names", + { + "event_id": event.event_id, + "room_id": event.room_id, + "name": event.name, + } + ) class RoomsTable(Table): From 0b51d970b4d53ef9c2c66e3537f56331f436ae76 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 10 Nov 2014 18:43:16 +0000 Subject: [PATCH 116/227] document up the current architecture a bit based on the workshop the other week --- docs/client-server/OLD_specification.rst | 2 +- docs/implementation-notes/architecture.rst | 65 +++++++++++++++++++ .../python_architecture.rst | 6 ++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 docs/implementation-notes/architecture.rst diff --git a/docs/client-server/OLD_specification.rst b/docs/client-server/OLD_specification.rst index 47fba5eeac..425ae57d93 100644 --- a/docs/client-server/OLD_specification.rst +++ b/docs/client-server/OLD_specification.rst @@ -4,7 +4,7 @@ Matrix Client-Server API .. WARNING:: - This specification is old. Please see /docs/specification.rst instead. + This specification is old. Please see matrix-doc/specification instead. diff --git a/docs/implementation-notes/architecture.rst b/docs/implementation-notes/architecture.rst new file mode 100644 index 0000000000..b447858202 --- /dev/null +++ b/docs/implementation-notes/architecture.rst @@ -0,0 +1,65 @@ +Synapse Architecture +==================== + +As of the end of Oct 2014, Synapse's overall architecture looks like:: + + Notifier + ^ | + | | + .------------|------. + | handlers/ | | + | v | + | Event*Handler<---------> rest/* <=> Client + | Rooms*Handler | + HSes <=> federation/* <==> FederationHandler | + | | PresenceHandler | + | | TypingHandler | + | '-------------------' + | | | + | state/* | + | | | + | v v + `--------------> storage/* + | + v + .----. + | DB | + '----' + +* Handlers: business logic of synapse itself. Follows a set contract of BaseHandler: + + * BaseHandler gives us onNewRoomEvent which: (TODO: flesh this out and make it less cryptic): + + * handle_state(event) + * auth(event) + * persist_event(event) + * notify notifier or federation(event) + + * PresenceHandler: use distributor to get EDUs out of Federation. Very + lightweight logic built on the distributor + * TypingHandler: use distributor to get EDUs out of Federation. Very + lightweight logic built on the distributor + * EventsHandler: handles the events stream... + * FederationHandler: - gets PDU from Federation Layer; turns into an event; + follows basehandler functionality. + * RoomsHandler: does all the room logic, including members - lots of classes in + RoomsHandler. + * ProfileHandler: talks to the storage to store/retrieve profile info. + +* EventFactory: generates events of particular event types. +* Notifier: Backs the events handler +* REST: Interfaces handlers and events to the outside world via HTTP/JSON. + Converts events back and forth from JSON. +* Federation: holds the HTTP client & server to talk to other servers. Does + replication to make sure there's nothing missing in the graph. Handles + reliability. Handles txns. +* Distributor: generic event bus. used for presence & typing only currently. + Notifier could be implemented using Distributor - so far we are only using for + things which actually /require/ dynamic pluggability however as it can + obfuscate the actual flow of control. +* Auth: helper singleton to say whether a given event is allowed to do a given + thing (TODO: put this on the diagram) +* State: helper singleton: does state conflict resolution. You give it an event + and it tells you if it actually updates the state or not, and annotates the + event up properly and handles merge conflict resolution. +* Storage: abstracts the storage engine. diff --git a/docs/implementation-notes/python_architecture.rst b/docs/implementation-notes/python_architecture.rst index 8beaa615d0..2a5a2613c4 100644 --- a/docs/implementation-notes/python_architecture.rst +++ b/docs/implementation-notes/python_architecture.rst @@ -1,3 +1,9 @@ +.. WARNING:: + These architecture notes are spectacularly old, and date back to when Synapse + was just federation code in isolation. This should be merged into the main + spec. + + = Server to Server = == Server to Server Stack == From f987393b32212f4fea4efa88a8d79a0d677e6f30 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 10 Nov 2014 21:56:52 +0000 Subject: [PATCH 117/227] moar boxes. --- docs/implementation-notes/architecture.rst | 39 ++++++++++++---------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/docs/implementation-notes/architecture.rst b/docs/implementation-notes/architecture.rst index b447858202..5b6463d78a 100644 --- a/docs/implementation-notes/architecture.rst +++ b/docs/implementation-notes/architecture.rst @@ -3,24 +3,27 @@ Synapse Architecture As of the end of Oct 2014, Synapse's overall architecture looks like:: - Notifier - ^ | - | | - .------------|------. - | handlers/ | | - | v | - | Event*Handler<---------> rest/* <=> Client - | Rooms*Handler | - HSes <=> federation/* <==> FederationHandler | - | | PresenceHandler | - | | TypingHandler | - | '-------------------' - | | | - | state/* | - | | | - | v v - `--------------> storage/* - | + synapse + .-----------------------------------------------------. + | Notifier | + | ^ | | + | | | | + | .------------|------. | + | | handlers/ | | | + | | v | | + | | Event*Handler<---------> rest/* <=> Client + | | Rooms*Handler | | + HSes <=> federation/* <==> FederationHandler | | + | | | PresenceHandler | | + | | | TypingHandler | | + | | '-------------------' | + | | | | | + | | state/* | | + | | | | | + | | v v | + | `--------------> storage/* | + | | | + '--------------------------|--------------------------' v .----. | DB | From f45a6a70041dfb38a8f3266d17d4b9269dca0bf7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 22:07:08 +0000 Subject: [PATCH 118/227] Fix RST sublist formatting bug --- docs/implementation-notes/architecture.rst | 30 +++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/implementation-notes/architecture.rst b/docs/implementation-notes/architecture.rst index 5b6463d78a..b8476ddbf6 100644 --- a/docs/implementation-notes/architecture.rst +++ b/docs/implementation-notes/architecture.rst @@ -31,23 +31,23 @@ As of the end of Oct 2014, Synapse's overall architecture looks like:: * Handlers: business logic of synapse itself. Follows a set contract of BaseHandler: - * BaseHandler gives us onNewRoomEvent which: (TODO: flesh this out and make it less cryptic): + - BaseHandler gives us onNewRoomEvent which: (TODO: flesh this out and make it less cryptic): - * handle_state(event) - * auth(event) - * persist_event(event) - * notify notifier or federation(event) + + handle_state(event) + + auth(event) + + persist_event(event) + + notify notifier or federation(event) - * PresenceHandler: use distributor to get EDUs out of Federation. Very - lightweight logic built on the distributor - * TypingHandler: use distributor to get EDUs out of Federation. Very - lightweight logic built on the distributor - * EventsHandler: handles the events stream... - * FederationHandler: - gets PDU from Federation Layer; turns into an event; - follows basehandler functionality. - * RoomsHandler: does all the room logic, including members - lots of classes in - RoomsHandler. - * ProfileHandler: talks to the storage to store/retrieve profile info. + - PresenceHandler: use distributor to get EDUs out of Federation. Very + lightweight logic built on the distributor + - TypingHandler: use distributor to get EDUs out of Federation. Very + lightweight logic built on the distributor + - EventsHandler: handles the events stream... + - FederationHandler: - gets PDU from Federation Layer; turns into an event; + follows basehandler functionality. + - RoomsHandler: does all the room logic, including members - lots of classes in + RoomsHandler. + - ProfileHandler: talks to the storage to store/retrieve profile info. * EventFactory: generates events of particular event types. * Notifier: Backs the events handler From 303b455965b3a9a4045fefa4f329ac805fb034e8 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 11 Nov 2014 04:35:16 +0000 Subject: [PATCH 119/227] trivial spacing fix --- docs/implementation-notes/architecture.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/implementation-notes/architecture.rst b/docs/implementation-notes/architecture.rst index b8476ddbf6..98050428b9 100644 --- a/docs/implementation-notes/architecture.rst +++ b/docs/implementation-notes/architecture.rst @@ -11,7 +11,7 @@ As of the end of Oct 2014, Synapse's overall architecture looks like:: | .------------|------. | | | handlers/ | | | | | v | | - | | Event*Handler<---------> rest/* <=> Client + | | Event*Handler <--------> rest/* <=> Client | | Rooms*Handler | | HSes <=> federation/* <==> FederationHandler | | | | | PresenceHandler | | From b765dc005ba22cb41e01439f1b07d567d2de68da Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 11 Nov 2014 04:39:16 +0000 Subject: [PATCH 120/227] major CSS overhaul to try to make things look a bit cleaner --- syweb/webclient/app.css | 154 +++++++++++++++++++----- syweb/webclient/img/attach.png | Bin 0 -> 473 bytes syweb/webclient/img/settings.png | Bin 0 -> 864 bytes syweb/webclient/img/video.png | Bin 0 -> 604 bytes syweb/webclient/img/voice.png | Bin 0 -> 659 bytes syweb/webclient/index.html | 2 + syweb/webclient/js/angular-peity.js | 69 +++++++++++ syweb/webclient/js/jquery.peity.min.js | 13 ++ syweb/webclient/mobile.css | 18 +++ syweb/webclient/room/room-controller.js | 19 ++- syweb/webclient/room/room.html | 131 +++++++++----------- 11 files changed, 302 insertions(+), 104 deletions(-) create mode 100644 syweb/webclient/img/attach.png create mode 100644 syweb/webclient/img/settings.png create mode 100644 syweb/webclient/img/video.png create mode 100644 syweb/webclient/img/voice.png create mode 100644 syweb/webclient/js/angular-peity.js create mode 100644 syweb/webclient/js/jquery.peity.min.js diff --git a/syweb/webclient/app.css b/syweb/webclient/app.css index be2a73872d..e2f99f3975 100755 --- a/syweb/webclient/app.css +++ b/syweb/webclient/app.css @@ -318,7 +318,7 @@ textarea, input { position: absolute; bottom: 0px; width: 100%; - height: 100px; + height: 70px; background-color: #f8f8f8; border-top: #aaa 1px solid; } @@ -326,7 +326,9 @@ textarea, input { #controls { max-width: 1280px; padding: 12px; + padding-right: 42px; margin: auto; + position: relative; } #buttonsCell { @@ -343,7 +345,17 @@ textarea, input { #mainInput { width: 100%; - resize: none; + padding: 5px; +} + +#attachButton { + position: absolute; + margin-top: 3px; + right: 0px; + background: url('img/attach.png'); + width: 25px; + height: 25px; + border: 0px; } .blink { @@ -415,7 +427,8 @@ textarea, input { .roomHeaderInfo { text-align: right; float: right; - margin-top: 15px; + margin-top: 0px; + margin-right: 30px; } /*** Room Info Dialog ***/ @@ -449,15 +462,32 @@ textarea, input { resize: vertical; } +/*** Control Buttons ***/ +#controlButtons { + float: right; + margin-right: -4px; + padding-bottom: 6px; +} + +.controlButton { + border: 0px; + width: 30px; + height: 30px; + padding-left: 3px; + padding-right: 3px; +} + /*** Participant list ***/ #usersTableWrapper { float: right; - width: 120px; + clear: right; + width: 100px; height: 100%; overflow-y: auto; } +/* #usersTable { width: 100%; border-collapse: collapse; @@ -473,36 +503,66 @@ textarea, input { position: relative; background-color: #000; } +*/ -.userAvatar .userAvatarImage { - position: absolute; - top: 0px; - object-fit: cover; - width: 100%; +.userAvatar { } +.userAvatarFrame { + border-radius: 46px; + width: 80px; + margin: auto; + position: relative; + border: 3px solid #aaa; + background-color: #aaa; +} + +.userAvatarImage { + border-radius: 40px; + text-align: center; + object-fit: cover; + display: block; +} + +/* .userAvatar .userAvatarGradient { position: absolute; bottom: 20px; width: 100%; } +*/ -.userAvatar .userName { - position: absolute; - color: #fff; - margin: 2px; - bottom: 0px; +.userName { + margin-top: 3px; + margin-bottom: 6px; + text-align: center; font-size: 12px; word-break: break-all; } -.userAvatar .userPowerLevel { +.userPowerLevel { position: absolute; - bottom: 0px; - height: 2px; + bottom: -1px; + height: 1px; background-color: #f00; } +.userPowerLevelBar { + display: inline; + position: absolute; + width: 2px; + height: 10px; +/* border: 1px solid #000; +*/ background-color: #aaa; +} + +.userPowerLevelMeter { + position: relative; + bottom: 0px; + background-color: #f00; +} + +/* .userPresence { text-align: center; font-size: 12px; @@ -510,12 +570,15 @@ textarea, input { background-color: #aaa; border-bottom: 1px #ddd solid; } +*/ .online { + border-color: #38AF00; background-color: #38AF00; } .unavailable { + border-color: #FFCC00; background-color: #FFCC00; } @@ -538,18 +601,21 @@ textarea, input { #messageTable td { padding: 0px; +/* border: 1px solid #888; */ } .leftBlock { - width: 14em; + width: 6em; word-wrap: break-word; vertical-align: top; background-color: #fff; - color: #888; + color: #aaa; font-weight: medium; font-size: 12px; text-align: right; +/* border-top: 1px #ddd solid; +*/ } .rightBlock { @@ -560,13 +626,24 @@ textarea, input { } .sender, .timestamp { - padding-right: 1em; - padding-left: 1em; - padding-top: 3px; +/* padding-top: 3px; +*/} + +.timestamp { + font-size: 10px; + color: #ccc; + height: 13px; + margin-top: 4px; +*/ transition-property: opacity; + transition-duration: 0.3s; } .sender { - margin-bottom: -3px; + font-size: 12px; +/* + margin-top: 5px; + margin-bottom: -9px; +*/ } .avatar { @@ -577,7 +654,11 @@ textarea, input { } .avatarImage { + position: relative; + top: 5px; object-fit: cover; + border-radius: 32px; + margin-top: 4px; } .emote { @@ -591,6 +672,7 @@ textarea, input { } .image { + border: 1px solid #888; display: block; max-width:320px; max-height:320px; @@ -603,19 +685,23 @@ textarea, input { } .bubble { +/* background-color: #eee; border: 1px solid #d8d8d8; - display: inline-block; margin-bottom: -1px; - max-width: 90%; - font-size: 14px; - word-wrap: break-word; padding-top: 7px; padding-bottom: 5px; + -webkit-text-size-adjust:100% + vertical-align: middle; +*/ + display: inline-block; + max-width: 90%; padding-left: 1em; padding-right: 1em; - vertical-align: middle; - -webkit-text-size-adjust:100% + padding-top: 2px; + padding-bottom: 2px; + font-size: 14px; + word-wrap: break-word; } .bubble img { @@ -623,9 +709,11 @@ textarea, input { max-height: auto; } +/* .differentUser td { padding-bottom: 5px ! important; } +*/ .mine { text-align: right; @@ -635,13 +723,15 @@ textarea, input { .text.membership .bubble, .mine .text.emote .bubble, .mine .text.membership .bubble - { +{ background-color: transparent ! important; border: 0px ! important; } .mine .text .bubble { +/* background-color: #f8f8ff ! important; +*/ text-align: left ! important; } @@ -701,6 +791,8 @@ textarea, input { overflow: hidden; white-space: nowrap; text-overflow: ellipsis; + padding-left: 0.5em; + padding-right: 0.5em; } .recentsRoom { @@ -751,7 +843,7 @@ textarea, input { padding-right: 10px; margin-right: 10px; height: 100%; - border-right: 1px solid #ddd; +/* border-right: 1px solid #ddd; */ overflow-y: auto; } diff --git a/syweb/webclient/img/attach.png b/syweb/webclient/img/attach.png new file mode 100644 index 0000000000000000000000000000000000000000..d95eabaf00b81c675e9d8a7d1f963b62474db1b1 GIT binary patch literal 473 zcmV;~0Ve*5P)P000>X1^@s6#OZ}&0000PbVXQnQ*UN; zcVTj606}DLVr3vnZDD6+Qe|Oed2z{QJOBUzdr3q=RCwBAT(DpP1Cc-uh(&>z6NtY9 z@oOM{M2toj0`dkxoD9T)KrD?#{wWZz2jaOvd;ymsO#lD?$0iOka1jun2jUzY7Qocy zK-De9rH+Y_kr7?Y42aJFaV!IAAPTC_48syMzC92h24V>Y@_+g!b7j8)r18qPN%n~6W zcA}9bAajM7ph=FIMwWogH3LPw3ZY0QRvt%UPyzXbhk>?$A7sg28e8&<1y@Om&FduO z6J+oR7tJ10j;wcnb0Meg}uV(xURgeY5A4xR)3she& zdIo30<6R}FtKJbX{2f%CJg!vALO^pa6oV=tV`vqPoCnSV@oqw11PCwy$U{(wnBPLL P00000NkvXXu0mjfrxLb_ literal 0 HcmV?d00001 diff --git a/syweb/webclient/img/settings.png b/syweb/webclient/img/settings.png new file mode 100644 index 0000000000000000000000000000000000000000..ac99fe402be68535a6b0ea8f0d6a6131d3bbff8b GIT binary patch literal 864 zcmV-m1E2hfP){&EeYwdB|>SS^w6uv7HUh8KqXM=!IKo?MRV|^1eAaf z5DG$}|1E#E<2XCJruM^Qb!X?ldH?_UI~%N4tCml@PuLW zRm0;u6Jaiwvr?&))oeDc*=**Fff)ab#iD)1_+&D1a|MnQ!b~WBO4u+h7o@O!K5wPd zY3F`C9=A!cIH>kezQJ5}sL>3bxhaJsC=n`fFOm3L?9ph{NdcZ43~eqW1Mx~gus+jgiTk%$vAR?Spa5 z*F}fz3X1!>5rnVX?ambq4B=BNSq&-1kz+%I`awQH+|!N1q1p(rwXj!(8o7a}oo+lH zTm8OP*I9sjr`9l0j(}`f5TKHR2uej4bFmXBKIP-eLY@(7ea!E}v8wO+4hwOTEw zdua?@r%;U1W49_63gkU0D^Zn+Zl@NYWXJN`34bn3N<>bv5Ht}v1yG19$cDqA$qhK( z%I{;S-XnZ=n*ryYPRCIKB7`1w0M+PTKKA*ZIC;v)+b@Jyrc;O&$)CiS-LU0>?c7$B8xB$5;%Gr$xqMF~@A0 zrg6t3kq8?Oht^?a)q+%wxcjtFC@@);9l@Bq)oT5`Gl*9dg{4v{77B$}v)SbB^IZeX zl?c?B89te0GC6??C>u1DyoHp>WM&CZP)uRI8ZKoErT z7B6TFf)zPB9~ekS{)0bXb5si8XSW*8_otP5ClOG0ww7A9?u(B z+TEjxe&K`X`{Vigp7%cQbD?ZDtAv^-mWTyngqY#>XX2B1Anu7rNwCp?{xM>c*dt

IY@mFBO$8?!^Ax?ZuJ1NFd1hVX59_ag1ZB@de+qerbdo)2zj3B72-N50t{2B)Q`48q2T@ifmVqd z@2xh8v!Gd~X||a{XhWYsXYBQDyiH65&5A~&J@t7s!CtO0lEID|oZw{|oQ+}wG=|rr zHXgK=3{T4QRXiSdw~^l#!x*7vVQde<*oN`=5N`Fjp@t+F*N5O+H#mn5T8_aLM4)gG zgu`LgY&P5a(1t$2VWIU5<2%X7a=EN5%TmQ+v8@kn=))MH?r=IGX}?Lb7xHmBomQn% z$^DL1wpuNfOeR%6pI5b7O=61DXc8;Ny*ri6K;T?1=Ncvw33uP(8f`*T#a>N)9%I$7 zk`v0CP@qbs(#=ElPE4T))D;L;y(K9GkfDJ07{qEjJ#-zkr7KTw0zqYPBr(Qnaqt6u zEl@^RAHPc + + diff --git a/syweb/webclient/js/angular-peity.js b/syweb/webclient/js/angular-peity.js new file mode 100644 index 0000000000..2acb647d91 --- /dev/null +++ b/syweb/webclient/js/angular-peity.js @@ -0,0 +1,69 @@ +var angularPeity = angular.module( 'angular-peity', [] ); + +$.fn.peity.defaults.pie = { + fill: ["#ff0000", "#aaaaaa"], + radius: 4, +} + +var buildChartDirective = function ( chartType ) { + return { + restrict: 'E', + scope: { + data: "=", + options: "=" + }, + link: function ( scope, element, attrs ) { + + var options = {}; + if ( scope.options ) { + options = scope.options; + } + + // N.B. live-binding to data by Matthew + scope.$watch('data', function () { + var span = document.createElement( 'span' ); + span.textContent = scope.data.join(); + + if ( !attrs.class ) { + span.className = ""; + } else { + span.className = attrs.class; + } + + if (element[0].nodeType === 8) { + element.replaceWith( span ); + } + else if (element[0].firstChild) { + element.empty(); + element[0].appendChild( span ); + } + else { + element[0].appendChild( span ); + } + + jQuery( span ).peity( chartType, options ); + }); + } + }; +}; + + +angularPeity.directive( 'pieChart', function () { + + return buildChartDirective( "pie" ); + +} ); + + +angularPeity.directive( 'barChart', function () { + + return buildChartDirective( "bar" ); + +} ); + + +angularPeity.directive( 'lineChart', function () { + + return buildChartDirective( "line" ); + +} ); diff --git a/syweb/webclient/js/jquery.peity.min.js b/syweb/webclient/js/jquery.peity.min.js new file mode 100644 index 0000000000..054b83c5d8 --- /dev/null +++ b/syweb/webclient/js/jquery.peity.min.js @@ -0,0 +1,13 @@ +// Peity jQuery plugin version 3.0.2 +// (c) 2014 Ben Pickles +// +// http://benpickles.github.io/peity +// +// Released under MIT license. +(function(h,w,i,v){var p=function(a,b){var d=w.createElementNS("http://www.w3.org/2000/svg",a);h(d).attr(b);return d},y="createElementNS"in w&&p("svg",{}).createSVGRect,e=h.fn.peity=function(a,b){y&&this.each(function(){var d=h(this),c=d.data("peity");c?(a&&(c.type=a),h.extend(c.opts,b)):(c=new x(d,a,h.extend({},e.defaults[a],b)),d.change(function(){c.draw()}).data("peity",c));c.draw()});return this},x=function(a,b,d){this.$el=a;this.type=b;this.opts=d},r=x.prototype;r.draw=function(){e.graphers[this.type].call(this, +this.opts)};r.fill=function(){var a=this.opts.fill;return h.isFunction(a)?a:function(b,d){return a[d%a.length]}};r.prepare=function(a,b){this.svg||this.$el.hide().after(this.svg=p("svg",{"class":"peity"}));return h(this.svg).empty().data("peity",this).attr({height:b,width:a})};r.values=function(){return h.map(this.$el.text().split(this.opts.delimiter),function(a){return parseFloat(a)})};e.defaults={};e.graphers={};e.register=function(a,b,d){this.defaults[a]=b;this.graphers[a]=d};e.register("pie", +{fill:["#ff9900","#fff4dd","#ffc66e"],radius:8},function(a){if(!a.delimiter){var b=this.$el.text().match(/[^0-9\.]/);a.delimiter=b?b[0]:","}b=this.values();if("/"==a.delimiter)var d=b[0],b=[d,i.max(0,b[1]-d)];for(var c=0,d=b.length,n=0;cj?m=q(i.min(d,0)):o=q(i.max(c,0)):u=1;u=o-m;0==u&&(u=1,0