From 4dc9e268e68235d12bc3cb2119226660121b6d5d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 27 Aug 2024 18:08:17 -0500 Subject: [PATCH] Add test for catch-up background update --- synapse/storage/prepare_database.py | 4 +- tests/storage/test_sliding_sync_tables.py | 105 ++++++++++++++++++++-- 2 files changed, 102 insertions(+), 7 deletions(-) diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 2527766e2d..31f99782ee 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -24,7 +24,6 @@ import os import re from collections import Counter from typing import ( - cast, Collection, Counter as CounterType, Generator, @@ -33,11 +32,11 @@ from typing import ( Optional, TextIO, Tuple, + cast, ) import attr -from synapse.util import Clock, json_encoder from synapse.config.homeserver import HomeServerConfig from synapse.storage.database import ( DatabasePool, @@ -49,6 +48,7 @@ from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3E from synapse.storage.schema import SCHEMA_COMPAT_VERSION, SCHEMA_VERSION from synapse.storage.types import Cursor from synapse.types import JsonDict +from synapse.util import json_encoder from synapse.util.iterutils import batch_iter logger = logging.getLogger(__name__) diff --git a/tests/storage/test_sliding_sync_tables.py b/tests/storage/test_sliding_sync_tables.py index 012176784c..23a0aee25c 100644 --- a/tests/storage/test_sliding_sync_tables.py +++ b/tests/storage/test_sliding_sync_tables.py @@ -34,6 +34,9 @@ from synapse.rest.client import login, room from synapse.server import HomeServer from synapse.storage.databases.main.events import DeltaState from synapse.storage.databases.main.events_bg_updates import _BackgroundUpdates +from synapse.storage.prepare_database import ( + _resolve_stale_data_in_sliding_sync_joined_rooms_table, +) from synapse.util import Clock from tests.test_utils.event_injection import create_event @@ -4176,8 +4179,14 @@ class SlidingSyncTablesBackgroundUpdatesTestCase(SlidingSyncTablesTestCaseBase): class SlidingSyncTablesCatchUpBackgroundUpdatesTestCase(SlidingSyncTablesTestCaseBase): """ - Test the background updates for catch-up after Synapse downgrade populate the `sliding_sync_joined_rooms` and - `sliding_sync_membership_snapshots` tables. + Test the background updates for catch-up after Synapse downgrade to populate the + `sliding_sync_joined_rooms` and `sliding_sync_membership_snapshots` tables. + + This to test the "catch-up" version of the background update vs the "normal" + background update to populate the tables with all of the historical data. Both + versions share the same background update but just serve different purposes. We + check if the "catch-up" version needs to run on start-up based on whether there have + been any changes to rooms that aren't reflected in the sliding sync tables. FIXME: This can be removed once we bump `SCHEMA_COMPAT_VERSION` and run the foreground update for @@ -4187,12 +4196,98 @@ class SlidingSyncTablesCatchUpBackgroundUpdatesTestCase(SlidingSyncTablesTestCas def test_joined_background_update_catch_up(self) -> None: """ - TODO + Test that new events while Synapse is downgraded (making + `sliding_sync_joined_rooms` stale) will be caught when Synapse is upgraded and + the catch-up routine is run. """ - pass + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + # Instead of testing with various levels of room state that should appear in the + # table, we're only using one room to keep this test simple. Because the + # underlying background update to populate these tables is the same as this + # catch-up routine, we are going to rely on + # `SlidingSyncTablesBackgroundUpdatesTestCase` to cover that logic. + room_id_with_info = self.helper.create_room_as(user1_id, tok=user1_tok) + + # Get a snapshot of the `sliding_sync_joined_rooms` table before we add some state + sliding_sync_joined_rooms_results_before_state = ( + self._get_sliding_sync_joined_rooms() + ) + self.assertIncludes( + set(sliding_sync_joined_rooms_results_before_state.keys()), + {room_id_with_info}, + exact=True, + ) + + # Add a room name + self.helper.send_state( + room_id_with_info, + EventTypes.Name, + {"name": "my super duper room"}, + tok=user1_tok, + ) + + # Make sure all of the background updates have finished before we start the + # catch-up. Even though it should work fine if the other background update is + # still running, we want to see the catch-up routine restore the progress + # correctly. + # + # We also don't want the normal background update messing with our results so we + # run this before we do our manual database clean-up to simulate new events + # being sent while Synapse was downgraded. + self.wait_for_background_updates() + + # Clean-up the `sliding_sync_joined_rooms` table as if the the room name + # never made it into the table. This is to simulate the room name event + # being sent while Synapse was downgraded. + self.get_success( + self.store.db_pool.simple_update( + table="sliding_sync_joined_rooms", + keyvalues={"room_id": room_id_with_info}, + updatevalues={ + # Clear the room name + "room_name": None, + # Reset the `event_stream_ordering` back to the value before the room name + "event_stream_ordering": sliding_sync_joined_rooms_results_before_state[ + room_id_with_info + ].event_stream_ordering, + }, + desc="sliding_sync_joined_rooms.test_joined_background_update_catch_up", + ) + ) + + # The function under test. It should clear out stale data and start the + # background update to catch-up on the missing data. + self.get_success( + self.store.db_pool.runInteraction( + "_resolve_stale_data_in_sliding_sync_joined_rooms_table", + _resolve_stale_data_in_sliding_sync_joined_rooms_table, + ) + ) + + # Ensure that the stale data is deleted from the table + sliding_sync_joined_rooms_results = self._get_sliding_sync_joined_rooms() + self.assertIncludes( + set(sliding_sync_joined_rooms_results.keys()), + set(), + exact=True, + ) + + # Wait for the catch-up background update to finish + self.store.db_pool.updates._all_done = False + self.wait_for_background_updates() + + # Ensure that the table is populated correctly after the catch-up background + # update finishes + sliding_sync_joined_rooms_results = self._get_sliding_sync_joined_rooms() + self.assertIncludes( + set(sliding_sync_joined_rooms_results.keys()), + {room_id_with_info}, + exact=True, + ) def test_membership_snapshots_background_update_catch_up(self) -> None: """ TODO """ - pass