mirrors/synapse
1
0
Fork 0
mirror of https://github.com/element-hq/synapse.git synced 2024-12-20 02:24:54 +03:00
Code Issues Projects Releases Packages Wiki Activity

concurrently_execute inserting

Browse source
This commit is contained in:
Eric Eastwood 2024-09-06 00:48:48 -05:00
parent 5894fc200c
commit b4fcbc9686
1 changed files with 77 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
synapse/storage/databases/main/events_bg_updates.py
View file

@ -1827,20 +1827,14 @@ class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseS
)
)
await concurrently_execute(
handle_room, [row[0] for row in rooms_to_update_rows], 10
)
rooms_to_update = [row[0] for row in rooms_to_update_rows]
await concurrently_execute(handle_room, rooms_to_update, 10)
def _fill_table_txn(txn: LoggingTransaction) -> None:
# Handle updating the `sliding_sync_joined_rooms` table
#
for (
room_id,
update_map,
) in joined_room_updates.items():
joined_room_stream_ordering_update = (
joined_room_stream_ordering_updates[room_id]
)
def _fill_table_for_room_txn(txn: LoggingTransaction, room_id: str) -> None:
update_map = joined_room_updates[room_id]
joined_room_stream_ordering_update = joined_room_stream_ordering_updates[
room_id
]
event_stream_ordering = (
joined_room_stream_ordering_update.most_recent_event_stream_ordering
)
@ -1876,6 +1870,8 @@ class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseS
+ "Raising exception so we can just try again."
)
# Handle updating the `sliding_sync_joined_rooms` table
#
# Since we fully insert rows into `sliding_sync_joined_rooms`, we can
# just do everything on insert and `ON CONFLICT DO NOTHING`.
#
@ -1894,21 +1890,27 @@ class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseS
},
)
# Now that we've processed all the room, we can remove them from the
async def fill_table_for_room(room_id: str) -> None:
await self.db_pool.runInteraction(
"sliding_sync_joined_rooms_bg_update", _fill_table_for_room_txn, room_id
)
# Since most of the time spent here is probably just latency to/from the
# database, let's just run this concurrently.
await concurrently_execute(fill_table_for_room, joined_room_updates.keys(), 10)
# Now that we've processed all of the rooms, we can remove them from the
# queue.
#
# Note: we need to remove all the rooms from the queue we pulled out
# from the DB, not just the ones we've processed above. Otherwise
# we'll simply keep pulling out the same rooms over and over again.
self.db_pool.simple_delete_many_batch_txn(
txn,
await self.db_pool.simple_delete_many(
table="sliding_sync_joined_rooms_to_recalculate",
keys=("room_id",),
values=rooms_to_update_rows,
)
await self.db_pool.runInteraction(
"sliding_sync_joined_rooms_bg_update", _fill_table_txn
column="room_id",
iterable=rooms_to_update,
keyvalues={},
desc="_sliding_sync_joined_rooms_bg_update: removing rooms that we processed",
)
return len(rooms_to_update_rows)