Remove old empty/redundant slaved stores. (#13349)

changelog.d/13349.misc

@@ -0,0 +1 @@
+Remove old base slaved store and de-duplicate cache ID generators. Contributed by Nick @ Beeper (@fizzadar).

synapse/app/admin_cmd.py

@@ -28,18 +28,22 @@ from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
 from synapse.events import EventBase
 from synapse.handlers.admin import ExfiltrationWriter
-from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
-from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
-from synapse.replication.slave.storage.deviceinbox import SlavedDeviceInboxStore
 from synapse.replication.slave.storage.devices import SlavedDeviceStore
 from synapse.replication.slave.storage.events import SlavedEventStore
 from synapse.replication.slave.storage.filtering import SlavedFilteringStore
 from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
-from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
-from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.server import HomeServer
 from synapse.storage.database import DatabasePool, LoggingDatabaseConnection
+from synapse.storage.databases.main.account_data import AccountDataWorkerStore
+from synapse.storage.databases.main.appservice import (
+    ApplicationServiceTransactionWorkerStore,
+    ApplicationServiceWorkerStore,
+)
+from synapse.storage.databases.main.deviceinbox import DeviceInboxWorkerStore
+from synapse.storage.databases.main.receipts import ReceiptsWorkerStore
+from synapse.storage.databases.main.registration import RegistrationWorkerStore
 from synapse.storage.databases.main.room import RoomWorkerStore
+from synapse.storage.databases.main.tags import TagsWorkerStore
 from synapse.types import StateMap
 from synapse.util import SYNAPSE_VERSION
 from synapse.util.logcontext import LoggingContext
@@ -48,15 +52,17 @@ logger = logging.getLogger("synapse.app.admin_cmd")
 
 
 class AdminCmdSlavedStore(
-    SlavedReceiptsStore,
-    SlavedAccountDataStore,
-    SlavedApplicationServiceStore,
-    SlavedRegistrationStore,
     SlavedFilteringStore,
-    SlavedDeviceInboxStore,
     SlavedDeviceStore,
     SlavedPushRuleStore,
     SlavedEventStore,
+    TagsWorkerStore,
+    DeviceInboxWorkerStore,
+    AccountDataWorkerStore,
+    ApplicationServiceTransactionWorkerStore,
+    ApplicationServiceWorkerStore,
+    RegistrationWorkerStore,
+    ReceiptsWorkerStore,
     RoomWorkerStore,
 ):
     def __init__(

synapse/app/generic_worker.py

@@ -48,19 +48,12 @@ from synapse.http.site import SynapseRequest, SynapseSite
 from synapse.logging.context import LoggingContext
 from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
 from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
-from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
-from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
-from synapse.replication.slave.storage.deviceinbox import SlavedDeviceInboxStore
 from synapse.replication.slave.storage.devices import SlavedDeviceStore
-from synapse.replication.slave.storage.directory import DirectoryStore
 from synapse.replication.slave.storage.events import SlavedEventStore
 from synapse.replication.slave.storage.filtering import SlavedFilteringStore
 from synapse.replication.slave.storage.keys import SlavedKeyStore
-from synapse.replication.slave.storage.profile import SlavedProfileStore
 from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
 from synapse.replication.slave.storage.pushers import SlavedPusherStore
-from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
-from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.rest.admin import register_servlets_for_media_repo
 from synapse.rest.client import (
     account_data,
@@ -99,8 +92,15 @@ from synapse.rest.key.v2 import KeyApiV2Resource
 from synapse.rest.synapse.client import build_synapse_client_resource_tree
 from synapse.rest.well_known import well_known_resource
 from synapse.server import HomeServer
+from synapse.storage.databases.main.account_data import AccountDataWorkerStore
+from synapse.storage.databases.main.appservice import (
+    ApplicationServiceTransactionWorkerStore,
+    ApplicationServiceWorkerStore,
+)
 from synapse.storage.databases.main.censor_events import CensorEventsStore
 from synapse.storage.databases.main.client_ips import ClientIpWorkerStore
+from synapse.storage.databases.main.deviceinbox import DeviceInboxWorkerStore
+from synapse.storage.databases.main.directory import DirectoryWorkerStore
 from synapse.storage.databases.main.e2e_room_keys import EndToEndRoomKeyStore
 from synapse.storage.databases.main.lock import LockStore
 from synapse.storage.databases.main.media_repository import MediaRepositoryStore
@@ -109,11 +109,15 @@ from synapse.storage.databases.main.monthly_active_users import (
     MonthlyActiveUsersWorkerStore,
 )
 from synapse.storage.databases.main.presence import PresenceStore
+from synapse.storage.databases.main.profile import ProfileWorkerStore
+from synapse.storage.databases.main.receipts import ReceiptsWorkerStore
+from synapse.storage.databases.main.registration import RegistrationWorkerStore
 from synapse.storage.databases.main.room import RoomWorkerStore
 from synapse.storage.databases.main.room_batch import RoomBatchStore
 from synapse.storage.databases.main.search import SearchStore
 from synapse.storage.databases.main.session import SessionStore
 from synapse.storage.databases.main.stats import StatsStore
+from synapse.storage.databases.main.tags import TagsWorkerStore
 from synapse.storage.databases.main.transactions import TransactionWorkerStore
 from synapse.storage.databases.main.ui_auth import UIAuthWorkerStore
 from synapse.storage.databases.main.user_directory import UserDirectoryStore
@@ -226,11 +230,11 @@ class GenericWorkerSlavedStore(
     UIAuthWorkerStore,
     EndToEndRoomKeyStore,
     PresenceStore,
-    SlavedDeviceInboxStore,
+    DeviceInboxWorkerStore,
     SlavedDeviceStore,
-    SlavedReceiptsStore,
     SlavedPushRuleStore,
-    SlavedAccountDataStore,
+    TagsWorkerStore,
+    AccountDataWorkerStore,
     SlavedPusherStore,
     CensorEventsStore,
     ClientIpWorkerStore,
@@ -238,14 +242,16 @@ class GenericWorkerSlavedStore(
     SlavedKeyStore,
     RoomWorkerStore,
     RoomBatchStore,
-    DirectoryStore,
-    SlavedApplicationServiceStore,
-    SlavedRegistrationStore,
-    SlavedProfileStore,
+    DirectoryWorkerStore,
+    ApplicationServiceTransactionWorkerStore,
+    ApplicationServiceWorkerStore,
+    ProfileWorkerStore,
     SlavedFilteringStore,
     MonthlyActiveUsersWorkerStore,
     MediaRepositoryStore,
     ServerMetricsStore,
+    ReceiptsWorkerStore,
+    RegistrationWorkerStore,
     SearchStore,
     TransactionWorkerStore,
     LockStore,

synapse/replication/slave/storage/account_data.py

@@ -1,21 +0,0 @@
-# Copyright 2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse.storage.databases.main.account_data import AccountDataWorkerStore
-from synapse.storage.databases.main.tags import TagsWorkerStore
-
-
-class SlavedAccountDataStore(TagsWorkerStore, AccountDataWorkerStore):
-    pass

synapse/replication/slave/storage/appservice.py

@@ -1,25 +0,0 @@
-# Copyright 2015, 2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse.storage.databases.main.appservice import (
-    ApplicationServiceTransactionWorkerStore,
-    ApplicationServiceWorkerStore,
-)
-
-
-class SlavedApplicationServiceStore(
-    ApplicationServiceTransactionWorkerStore, ApplicationServiceWorkerStore
-):
-    pass

synapse/replication/slave/storage/deviceinbox.py

@@ -1,19 +0,0 @@
-# Copyright 2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse.storage.databases.main.deviceinbox import DeviceInboxWorkerStore
-
-
-class SlavedDeviceInboxStore(DeviceInboxWorkerStore):
-    pass

synapse/replication/slave/storage/directory.py

@@ -1,19 +0,0 @@
-# Copyright 2015, 2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse.storage.databases.main.directory import DirectoryWorkerStore
-
-
-class DirectoryStore(DirectoryWorkerStore):
-    pass

synapse/replication/slave/storage/profile.py

@@ -1,19 +0,0 @@
-# Copyright 2018 New Vector Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse.storage.databases.main.profile import ProfileWorkerStore
-
-
-class SlavedProfileStore(ProfileWorkerStore):
-    pass

synapse/replication/slave/storage/receipts.py

@@ -1,20 +0,0 @@
-# Copyright 2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse.storage.databases.main.receipts import ReceiptsWorkerStore
-
-
-class SlavedReceiptsStore(ReceiptsWorkerStore):
-    pass

synapse/replication/slave/storage/registration.py

@@ -1,19 +0,0 @@
-# Copyright 2015, 2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse.storage.databases.main.registration import RegistrationWorkerStore
-
-
-class SlavedRegistrationStore(RegistrationWorkerStore):
-    pass

tests/replication/slave/storage/test_account_data.py

@@ -1,42 +0,0 @@
-# Copyright 2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
-
-from ._base import BaseSlavedStoreTestCase
-
-USER_ID = "@feeling:blue"
-TYPE = "my.type"
-
-
-class SlavedAccountDataStoreTestCase(BaseSlavedStoreTestCase):
-
-    STORE_TYPE = SlavedAccountDataStore
-
-    def test_user_account_data(self):
-        self.get_success(
-            self.master_store.add_account_data_for_user(USER_ID, TYPE, {"a": 1})
-        )
-        self.replicate()
-        self.check(
-            "get_global_account_data_by_type_for_user", [USER_ID, TYPE], {"a": 1}
-        )
-
-        self.get_success(
-            self.master_store.add_account_data_for_user(USER_ID, TYPE, {"a": 2})
-        )
-        self.replicate()
-        self.check(
-            "get_global_account_data_by_type_for_user", [USER_ID, TYPE], {"a": 2}
-        )

tests/storage/test_receipts.py

@@ -13,23 +13,21 @@
 # limitations under the License.
 
 from synapse.api.constants import ReceiptTypes
-from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
 from synapse.types import UserID, create_requester
 
 from tests.test_utils.event_injection import create_event
-
-from ._base import BaseSlavedStoreTestCase
+from tests.unittest import HomeserverTestCase
 
 OTHER_USER_ID = "@other:test"
 OUR_USER_ID = "@our:test"
 
 
-class SlavedReceiptTestCase(BaseSlavedStoreTestCase):
-
-    STORE_TYPE = SlavedReceiptsStore
-
+class ReceiptTestCase(HomeserverTestCase):
     def prepare(self, reactor, clock, homeserver):
         super().prepare(reactor, clock, homeserver)
+
+        self.store = homeserver.get_datastores().main
+
         self.room_creator = homeserver.get_room_creation_handler()
         self.persist_event_storage_controller = (
             self.hs.get_storage_controllers().persistence
@@ -87,14 +85,14 @@ class SlavedReceiptTestCase(BaseSlavedStoreTestCase):
 
     def test_return_empty_with_no_data(self):
         res = self.get_success(
-            self.master_store.get_receipts_for_user(
+            self.store.get_receipts_for_user(
                 OUR_USER_ID, [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE]
             )
         )
         self.assertEqual(res, {})
 
         res = self.get_success(
-            self.master_store.get_receipts_for_user_with_orderings(
+            self.store.get_receipts_for_user_with_orderings(
                 OUR_USER_ID,
                 [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE],
             )
@@ -102,7 +100,7 @@ class SlavedReceiptTestCase(BaseSlavedStoreTestCase):
         self.assertEqual(res, {})
 
         res = self.get_success(
-            self.master_store.get_last_receipt_event_id_for_user(
+            self.store.get_last_receipt_event_id_for_user(
                 OUR_USER_ID,
                 self.room_id1,
                 [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE],
@@ -121,20 +119,20 @@ class SlavedReceiptTestCase(BaseSlavedStoreTestCase):
 
         # Send public read receipt for the first event
         self.get_success(
-            self.master_store.insert_receipt(
+            self.store.insert_receipt(
                 self.room_id1, ReceiptTypes.READ, OUR_USER_ID, [event1_1_id], {}
             )
         )
         # Send private read receipt for the second event
         self.get_success(
-            self.master_store.insert_receipt(
+            self.store.insert_receipt(
                 self.room_id1, ReceiptTypes.READ_PRIVATE, OUR_USER_ID, [event1_2_id], {}
             )
         )
 
         # Test we get the latest event when we want both private and public receipts
         res = self.get_success(
-            self.master_store.get_receipts_for_user(
+            self.store.get_receipts_for_user(
                 OUR_USER_ID, [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE]
             )
         )
@@ -142,26 +140,24 @@ class SlavedReceiptTestCase(BaseSlavedStoreTestCase):
 
         # Test we get the older event when we want only public receipt
         res = self.get_success(
-            self.master_store.get_receipts_for_user(OUR_USER_ID, [ReceiptTypes.READ])
+            self.store.get_receipts_for_user(OUR_USER_ID, [ReceiptTypes.READ])
         )
         self.assertEqual(res, {self.room_id1: event1_1_id})
 
         # Test we get the latest event when we want only the public receipt
         res = self.get_success(
-            self.master_store.get_receipts_for_user(
-                OUR_USER_ID, [ReceiptTypes.READ_PRIVATE]
-            )
+            self.store.get_receipts_for_user(OUR_USER_ID, [ReceiptTypes.READ_PRIVATE])
         )
         self.assertEqual(res, {self.room_id1: event1_2_id})
 
         # Test receipt updating
         self.get_success(
-            self.master_store.insert_receipt(
+            self.store.insert_receipt(
                 self.room_id1, ReceiptTypes.READ, OUR_USER_ID, [event1_2_id], {}
             )
         )
         res = self.get_success(
-            self.master_store.get_receipts_for_user(OUR_USER_ID, [ReceiptTypes.READ])
+            self.store.get_receipts_for_user(OUR_USER_ID, [ReceiptTypes.READ])
         )
         self.assertEqual(res, {self.room_id1: event1_2_id})
 
@@ -172,12 +168,12 @@ class SlavedReceiptTestCase(BaseSlavedStoreTestCase):
 
         # Test new room is reflected in what the method returns
         self.get_success(
-            self.master_store.insert_receipt(
+            self.store.insert_receipt(
                 self.room_id2, ReceiptTypes.READ_PRIVATE, OUR_USER_ID, [event2_1_id], {}
             )
         )
         res = self.get_success(
-            self.master_store.get_receipts_for_user(
+            self.store.get_receipts_for_user(
                 OUR_USER_ID, [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE]
             )
         )
@@ -194,20 +190,20 @@ class SlavedReceiptTestCase(BaseSlavedStoreTestCase):
 
         # Send public read receipt for the first event
         self.get_success(
-            self.master_store.insert_receipt(
+            self.store.insert_receipt(
                 self.room_id1, ReceiptTypes.READ, OUR_USER_ID, [event1_1_id], {}
             )
         )
         # Send private read receipt for the second event
         self.get_success(
-            self.master_store.insert_receipt(
+            self.store.insert_receipt(
                 self.room_id1, ReceiptTypes.READ_PRIVATE, OUR_USER_ID, [event1_2_id], {}
             )
         )
 
         # Test we get the latest event when we want both private and public receipts
         res = self.get_success(
-            self.master_store.get_last_receipt_event_id_for_user(
+            self.store.get_last_receipt_event_id_for_user(
                 OUR_USER_ID,
                 self.room_id1,
                 [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE],
@@ -217,7 +213,7 @@ class SlavedReceiptTestCase(BaseSlavedStoreTestCase):
 
         # Test we get the older event when we want only public receipt
         res = self.get_success(
-            self.master_store.get_last_receipt_event_id_for_user(
+            self.store.get_last_receipt_event_id_for_user(
                 OUR_USER_ID, self.room_id1, [ReceiptTypes.READ]
             )
         )
@@ -225,7 +221,7 @@ class SlavedReceiptTestCase(BaseSlavedStoreTestCase):
 
         # Test we get the latest event when we want only the private receipt
         res = self.get_success(
-            self.master_store.get_last_receipt_event_id_for_user(
+            self.store.get_last_receipt_event_id_for_user(
                 OUR_USER_ID, self.room_id1, [ReceiptTypes.READ_PRIVATE]
             )
         )
@@ -233,12 +229,12 @@ class SlavedReceiptTestCase(BaseSlavedStoreTestCase):
 
         # Test receipt updating
         self.get_success(
-            self.master_store.insert_receipt(
+            self.store.insert_receipt(
                 self.room_id1, ReceiptTypes.READ, OUR_USER_ID, [event1_2_id], {}
             )
         )
         res = self.get_success(
-            self.master_store.get_last_receipt_event_id_for_user(
+            self.store.get_last_receipt_event_id_for_user(
                 OUR_USER_ID, self.room_id1, [ReceiptTypes.READ]
             )
         )
@@ -251,12 +247,12 @@ class SlavedReceiptTestCase(BaseSlavedStoreTestCase):
 
         # Test new room is reflected in what the method returns
         self.get_success(
-            self.master_store.insert_receipt(
+            self.store.insert_receipt(
                 self.room_id2, ReceiptTypes.READ_PRIVATE, OUR_USER_ID, [event2_1_id], {}
             )
         )
         res = self.get_success(
-            self.master_store.get_last_receipt_event_id_for_user(
+            self.store.get_last_receipt_event_id_for_user(
                 OUR_USER_ID,
                 self.room_id2,
                 [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE],