Merge pull request #93 from matrix-org/application-services-exclusive

Application services exclusive flag support

UPGRADE.rst

@@ -12,6 +12,10 @@ Servers which use captchas will need to add their public key to::
 This is required in order to support registration fallback (typically used on
 mobile devices).
 
+The format of stored application services has changed in Synapse. You will need
+to run ``python upgrade_appservice_db.py <database file path>`` to convert to 
+the new format.
+
 Upgrading to v0.7.0
 ===================
 

scripts/upgrade_appservice_db.py

@@ -0,0 +1,54 @@
+from synapse.storage import read_schema
+import argparse
+import json
+import sqlite3
+
+
+def do_other_deltas(cursor):
+    cursor.execute("PRAGMA user_version")
+    row = cursor.fetchone()
+
+    if row and row[0]:
+        user_version = row[0]
+        # Run every version since after the current version.
+        for v in range(user_version + 1, 10):
+            print "Running delta: %d" % (v,)
+            sql_script = read_schema("delta/v%d" % (v,))
+            cursor.executescript(sql_script)
+
+
+def update_app_service_table(cur):
+    cur.execute("SELECT id, regex FROM application_services_regex")
+    for row in cur.fetchall():
+        try:
+            print "checking %s..." % row[0]
+            json.loads(row[1])
+        except ValueError:
+            # row isn't in json, make it so.
+            string_regex = row[1]
+            new_regex = json.dumps({
+                "regex": string_regex,
+                "exclusive": True
+            })
+            cur.execute(
+                "UPDATE application_services_regex SET regex=? WHERE id=?",
+                (new_regex, row[0])
+            )
+
+
+def main(dbname):
+    con = sqlite3.connect(dbname)
+    cur = con.cursor()
+    do_other_deltas(cur)
+    update_app_service_table(cur)
+    cur.execute("PRAGMA user_version = 14")
+    cur.close()
+    con.commit()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("database")
+    args = parser.parse_args()
+
+    main(args.database)

synapse/appservice/__init__.py

@@ -46,22 +46,34 @@ class ApplicationService(object):
     def _check_namespaces(self, namespaces):
         # Sanity check that it is of the form:
         # {
-        #   users: ["regex",...],
+        #   users: [ {regex: "[A-z]+.*", exclusive: true}, ...],
-        #   aliases: ["regex",...],
+        #   aliases: [ {regex: "[A-z]+.*", exclusive: true}, ...],
-        #   rooms: ["regex",...],
+        #   rooms: [ {regex: "[A-z]+.*", exclusive: true}, ...],
         # }
         if not namespaces:
             return None
 
         for ns in ApplicationService.NS_LIST:
+            if ns not in namespaces:
+                namespaces[ns] = []
+                continue
+
             if type(namespaces[ns]) != list:
-                raise ValueError("Bad namespace value for '%s'", ns)
+                raise ValueError("Bad namespace value for '%s'" % ns)
-            for regex in namespaces[ns]:
+            for regex_obj in namespaces[ns]:
-                if not isinstance(regex, basestring):
+                if not isinstance(regex_obj, dict):
-                    raise ValueError("Expected string regex for ns '%s'", ns)
+                    raise ValueError("Expected dict regex for ns '%s'" % ns)
+                if not isinstance(regex_obj.get("exclusive"), bool):
+                    raise ValueError(
+                        "Expected bool for 'exclusive' in ns '%s'" % ns
+                    )
+                if not isinstance(regex_obj.get("regex"), basestring):
+                    raise ValueError(
+                        "Expected string for 'regex' in ns '%s'" % ns
+                    )
         return namespaces
 
-    def _matches_regex(self, test_string, namespace_key):
+    def _matches_regex(self, test_string, namespace_key, return_obj=False):
         if not isinstance(test_string, basestring):
             logger.error(
                 "Expected a string to test regex against, but got %s",
@@ -69,11 +81,19 @@ class ApplicationService(object):
             )
             return False
 
-        for regex in self.namespaces[namespace_key]:
+        for regex_obj in self.namespaces[namespace_key]:
-            if re.match(regex, test_string):
+            if re.match(regex_obj["regex"], test_string):
+                if return_obj:
+                    return regex_obj
                 return True
         return False
 
+    def _is_exclusive(self, ns_key, test_string):
+        regex_obj = self._matches_regex(test_string, ns_key, return_obj=True)
+        if regex_obj:
+            return regex_obj["exclusive"]
+        return False
+
     def _matches_user(self, event, member_list):
         if (hasattr(event, "sender") and
                 self.is_interested_in_user(event.sender)):
@@ -143,5 +163,14 @@ class ApplicationService(object):
     def is_interested_in_room(self, room_id):
         return self._matches_regex(room_id, ApplicationService.NS_ROOMS)
 
+    def is_exclusive_user(self, user_id):
+        return self._is_exclusive(ApplicationService.NS_USERS, user_id)
+
+    def is_exclusive_alias(self, alias):
+        return self._is_exclusive(ApplicationService.NS_ALIASES, alias)
+
+    def is_exclusive_room(self, room_id):
+        return self._is_exclusive(ApplicationService.NS_ROOMS, room_id)
+
     def __str__(self):
         return "ApplicationService: %s" % (self.__dict__,)

synapse/handlers/directory.py

@@ -232,13 +232,23 @@ class DirectoryHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def can_modify_alias(self, alias, user_id=None):
+        # Any application service "interested" in an alias they are regexing on
+        # can modify the alias.
+        # Users can only modify the alias if ALL the interested services have
+        # non-exclusive locks on the alias (or there are no interested services)
         services = yield self.store.get_app_services()
         interested_services = [
             s for s in services if s.is_interested_in_alias(alias.to_string())
         ]
+
         for service in interested_services:
             if user_id == service.sender:
-                # this user IS the app service
+                # this user IS the app service so they can do whatever they like
                 defer.returnValue(True)
                 return
-        defer.returnValue(len(interested_services) == 0)
+            elif service.is_exclusive_alias(alias.to_string()):
+                # another service has an exclusive lock on this alias.
+                defer.returnValue(False)
+                return
+        # either no interested services, or no service with an exclusive lock
+        defer.returnValue(True)

synapse/handlers/register.py

@@ -201,11 +201,12 @@ class RegistrationHandler(BaseHandler):
         interested_services = [
             s for s in services if s.is_interested_in_user(user_id)
         ]
-        if len(interested_services) > 0:
+        for service in interested_services:
-            raise SynapseError(
+            if service.is_exclusive_user(user_id):
-                400, "This user ID is reserved by an application service.",
+                raise SynapseError(
-                errcode=Codes.EXCLUSIVE
+                    400, "This user ID is reserved by an application service.",
-            )
+                    errcode=Codes.EXCLUSIVE
+                )
 
     def _generate_token(self, user_id):
         # urlsafe variant uses _ and - so use . as the separator and replace

synapse/rest/appservice/v1/register.py

@@ -48,18 +48,12 @@ class RegisterRestServlet(AppServiceRestServlet):
                 400, "Missed required keys: as_token(str) / url(str)."
             )
 
-        namespaces = {
+        try:
-            "users": [],
+            app_service = ApplicationService(
-            "rooms": [],
+                as_token, as_url, params["namespaces"]
-            "aliases": []
+            )
-        }
+        except ValueError as e:
-
+            raise SynapseError(400, e.message)
-        if "namespaces" in params:
-            self._parse_namespace(namespaces, params["namespaces"], "users")
-            self._parse_namespace(namespaces, params["namespaces"], "rooms")
-            self._parse_namespace(namespaces, params["namespaces"], "aliases")
-
-        app_service = ApplicationService(as_token, as_url, namespaces)
 
         app_service = yield self.handler.register(app_service)
         hs_token = app_service.hs_token
@@ -68,23 +62,6 @@ class RegisterRestServlet(AppServiceRestServlet):
             "hs_token": hs_token
         }))
 
-    def _parse_namespace(self, target_ns, origin_ns, ns):
-        if ns not in target_ns or ns not in origin_ns:
-            return  # nothing to parse / map through to.
-
-        possible_regex_list = origin_ns[ns]
-        if not type(possible_regex_list) == list:
-            raise SynapseError(400, "Namespace %s isn't an array." % ns)
-
-        for regex in possible_regex_list:
-            if not isinstance(regex, basestring):
-                raise SynapseError(
-                    400, "Regex '%s' isn't a string in namespace %s" %
-                    (regex, ns)
-                )
-
-        target_ns[ns] = origin_ns[ns]
-
 
 class UnregisterRestServlet(AppServiceRestServlet):
     """Handles AS registration with the home server.

synapse/storage/__init__.py

@@ -74,7 +74,7 @@ SCHEMAS = [
 
 # Remember to update this number every time an incompatible change is made to
 # database schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 13
+SCHEMA_VERSION = 14
 
 dir_path = os.path.abspath(os.path.dirname(__file__))
 

synapse/storage/appservice.py

@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
+import simplejson
+from simplejson import JSONDecodeError
 from twisted.internet import defer
 
 from synapse.api.constants import Membership
@@ -25,12 +27,18 @@ from ._base import SQLBaseStore
 logger = logging.getLogger(__name__)
 
 
+def log_failure(failure):
+    logger.error("Failed to detect application services: %s", failure.value)
+    logger.error(failure.getTraceback())
+
+
 class ApplicationServiceStore(SQLBaseStore):
 
     def __init__(self, hs):
         super(ApplicationServiceStore, self).__init__(hs)
         self.services_cache = []
         self.cache_defer = self._populate_cache()
+        self.cache_defer.addErrback(log_failure)
 
     @defer.inlineCallbacks
     def unregister_app_service(self, token):
@@ -130,11 +138,11 @@ class ApplicationServiceStore(SQLBaseStore):
         )
         for (ns_int, ns_str) in enumerate(ApplicationService.NS_LIST):
             if ns_str in service.namespaces:
-                for regex in service.namespaces[ns_str]:
+                for regex_obj in service.namespaces[ns_str]:
                     txn.execute(
                         "INSERT INTO application_services_regex("
                         "as_id, namespace, regex) values(?,?,?)",
-                        (as_id, ns_int, regex)
+                        (as_id, ns_int, simplejson.dumps(regex_obj))
                     )
         return True
 
@@ -311,10 +319,12 @@ class ApplicationServiceStore(SQLBaseStore):
             try:
                 services[as_token]["namespaces"][
                     ApplicationService.NS_LIST[ns_int]].append(
-                    res["regex"]
+                    simplejson.loads(res["regex"])
                 )
             except IndexError:
                 logger.error("Bad namespace enum '%s'. %s", ns_int, res)
+            except JSONDecodeError:
+                logger.error("Bad regex object '%s'", res["regex"])
 
         # TODO get last successful txn id f.e. service
         for service in services.values():

tests/appservice/test_appservice.py

@@ -18,6 +18,13 @@ from mock import Mock, PropertyMock
 from tests import unittest
 
 
+def _regex(regex, exclusive=True):
+    return {
+        "regex": regex,
+        "exclusive": exclusive
+    }
+
+
 class ApplicationServiceTestCase(unittest.TestCase):
 
     def setUp(self):
@@ -36,21 +43,21 @@ class ApplicationServiceTestCase(unittest.TestCase):
 
     def test_regex_user_id_prefix_match(self):
         self.service.namespaces[ApplicationService.NS_USERS].append(
-            "@irc_.*"
+            _regex("@irc_.*")
         )
         self.event.sender = "@irc_foobar:matrix.org"
         self.assertTrue(self.service.is_interested(self.event))
 
     def test_regex_user_id_prefix_no_match(self):
         self.service.namespaces[ApplicationService.NS_USERS].append(
-            "@irc_.*"
+            _regex("@irc_.*")
         )
         self.event.sender = "@someone_else:matrix.org"
         self.assertFalse(self.service.is_interested(self.event))
 
     def test_regex_room_member_is_checked(self):
         self.service.namespaces[ApplicationService.NS_USERS].append(
-            "@irc_.*"
+            _regex("@irc_.*")
         )
         self.event.sender = "@someone_else:matrix.org"
         self.event.type = "m.room.member"
@@ -59,30 +66,78 @@ class ApplicationServiceTestCase(unittest.TestCase):
 
     def test_regex_room_id_match(self):
         self.service.namespaces[ApplicationService.NS_ROOMS].append(
-            "!some_prefix.*some_suffix:matrix.org"
+            _regex("!some_prefix.*some_suffix:matrix.org")
         )
         self.event.room_id = "!some_prefixs0m3th1nGsome_suffix:matrix.org"
         self.assertTrue(self.service.is_interested(self.event))
 
     def test_regex_room_id_no_match(self):
         self.service.namespaces[ApplicationService.NS_ROOMS].append(
-            "!some_prefix.*some_suffix:matrix.org"
+            _regex("!some_prefix.*some_suffix:matrix.org")
         )
         self.event.room_id = "!XqBunHwQIXUiqCaoxq:matrix.org"
         self.assertFalse(self.service.is_interested(self.event))
 
     def test_regex_alias_match(self):
         self.service.namespaces[ApplicationService.NS_ALIASES].append(
-            "#irc_.*:matrix.org"
+            _regex("#irc_.*:matrix.org")
         )
         self.assertTrue(self.service.is_interested(
             self.event,
             aliases_for_event=["#irc_foobar:matrix.org", "#athing:matrix.org"]
         ))
 
+    def test_non_exclusive_alias(self):
+        self.service.namespaces[ApplicationService.NS_ALIASES].append(
+            _regex("#irc_.*:matrix.org", exclusive=False)
+        )
+        self.assertFalse(self.service.is_exclusive_alias(
+            "#irc_foobar:matrix.org"
+        ))
+
+    def test_non_exclusive_room(self):
+        self.service.namespaces[ApplicationService.NS_ROOMS].append(
+            _regex("!irc_.*:matrix.org", exclusive=False)
+        )
+        self.assertFalse(self.service.is_exclusive_room(
+            "!irc_foobar:matrix.org"
+        ))
+
+    def test_non_exclusive_user(self):
+        self.service.namespaces[ApplicationService.NS_USERS].append(
+            _regex("@irc_.*:matrix.org", exclusive=False)
+        )
+        self.assertFalse(self.service.is_exclusive_user(
+            "@irc_foobar:matrix.org"
+        ))
+
+    def test_exclusive_alias(self):
+        self.service.namespaces[ApplicationService.NS_ALIASES].append(
+            _regex("#irc_.*:matrix.org", exclusive=True)
+        )
+        self.assertTrue(self.service.is_exclusive_alias(
+            "#irc_foobar:matrix.org"
+        ))
+
+    def test_exclusive_user(self):
+        self.service.namespaces[ApplicationService.NS_USERS].append(
+            _regex("@irc_.*:matrix.org", exclusive=True)
+        )
+        self.assertTrue(self.service.is_exclusive_user(
+            "@irc_foobar:matrix.org"
+        ))
+
+    def test_exclusive_room(self):
+        self.service.namespaces[ApplicationService.NS_ROOMS].append(
+            _regex("!irc_.*:matrix.org", exclusive=True)
+        )
+        self.assertTrue(self.service.is_exclusive_room(
+            "!irc_foobar:matrix.org"
+        ))
+
     def test_regex_alias_no_match(self):
         self.service.namespaces[ApplicationService.NS_ALIASES].append(
-            "#irc_.*:matrix.org"
+            _regex("#irc_.*:matrix.org")
         )
         self.assertFalse(self.service.is_interested(
             self.event,
@@ -91,10 +146,10 @@ class ApplicationServiceTestCase(unittest.TestCase):
 
     def test_regex_multiple_matches(self):
         self.service.namespaces[ApplicationService.NS_ALIASES].append(
-            "#irc_.*:matrix.org"
+            _regex("#irc_.*:matrix.org")
         )
         self.service.namespaces[ApplicationService.NS_USERS].append(
-            "@irc_.*"
+            _regex("@irc_.*")
         )
         self.event.sender = "@irc_foobar:matrix.org"
         self.assertTrue(self.service.is_interested(
@@ -104,10 +159,10 @@ class ApplicationServiceTestCase(unittest.TestCase):
 
     def test_restrict_to_rooms(self):
         self.service.namespaces[ApplicationService.NS_ROOMS].append(
-            "!flibble_.*:matrix.org"
+            _regex("!flibble_.*:matrix.org")
         )
         self.service.namespaces[ApplicationService.NS_USERS].append(
-            "@irc_.*"
+            _regex("@irc_.*")
         )
         self.event.sender = "@irc_foobar:matrix.org"
         self.event.room_id = "!wibblewoo:matrix.org"
@@ -118,10 +173,10 @@ class ApplicationServiceTestCase(unittest.TestCase):
 
     def test_restrict_to_aliases(self):
         self.service.namespaces[ApplicationService.NS_ALIASES].append(
-            "#xmpp_.*:matrix.org"
+            _regex("#xmpp_.*:matrix.org")
         )
         self.service.namespaces[ApplicationService.NS_USERS].append(
-            "@irc_.*"
+            _regex("@irc_.*")
         )
         self.event.sender = "@irc_foobar:matrix.org"
         self.assertFalse(self.service.is_interested(
@@ -132,10 +187,10 @@ class ApplicationServiceTestCase(unittest.TestCase):
 
     def test_restrict_to_senders(self):
         self.service.namespaces[ApplicationService.NS_ALIASES].append(
-            "#xmpp_.*:matrix.org"
+            _regex("#xmpp_.*:matrix.org")
         )
         self.service.namespaces[ApplicationService.NS_USERS].append(
-            "@irc_.*"
+            _regex("@irc_.*")
         )
         self.event.sender = "@xmpp_foobar:matrix.org"
         self.assertFalse(self.service.is_interested(
@@ -146,7 +201,7 @@ class ApplicationServiceTestCase(unittest.TestCase):
 
     def test_member_list_match(self):
         self.service.namespaces[ApplicationService.NS_USERS].append(
-            "@irc_.*"
+            _regex("@irc_.*")
         )
         join_list = [
             Mock(

tests/storage/test_appservice.py

@@ -50,9 +50,15 @@ class ApplicationServiceStoreTestCase(unittest.TestCase):
     def test_update_and_retrieval_of_service(self):
         url = "https://matrix.org/appservices/foobar"
         hs_token = "hstok"
-        user_regex = ["@foobar_.*:matrix.org"]
+        user_regex = [
-        alias_regex = ["#foobar_.*:matrix.org"]
+            {"regex": "@foobar_.*:matrix.org", "exclusive": True}
-        room_regex = []
+        ]
+        alias_regex = [
+            {"regex": "#foobar_.*:matrix.org", "exclusive": False}
+        ]
+        room_regex = [
+
+        ]
         service = ApplicationService(
             url=url, hs_token=hs_token, token=self.as_token, namespaces={
                 ApplicationService.NS_USERS: user_regex,