mirrors/synapse
1
0
Fork 0
mirror of https://github.com/element-hq/synapse.git synced 2024-12-20 10:55:09 +03:00
Code Issues Projects Releases Packages Wiki Activity

concurrently_execute inserting

Browse source
This commit is contained in:
Eric Eastwood 2024-09-06 00:48:48 -05:00
parent 5894fc200c
commit b4fcbc9686
1 changed files with 77 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
synapse/storage/databases/main/events_bg_updates.py
View file

@ -1827,20 +1827,14 @@ class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseS
) )
) )
await concurrently_execute( rooms_to_update = [row[0] for row in rooms_to_update_rows]
handle_room, [row[0] for row in rooms_to_update_rows], 10 await concurrently_execute(handle_room, rooms_to_update, 10)
)
def _fill_table_txn(txn: LoggingTransaction) -> None: def _fill_table_for_room_txn(txn: LoggingTransaction, room_id: str) -> None:
# Handle updating the `sliding_sync_joined_rooms` table update_map = joined_room_updates[room_id]
# joined_room_stream_ordering_update = joined_room_stream_ordering_updates[
for ( room_id
room_id, ]
update_map,
) in joined_room_updates.items():
joined_room_stream_ordering_update = (
joined_room_stream_ordering_updates[room_id]
)
event_stream_ordering = ( event_stream_ordering = (
joined_room_stream_ordering_update.most_recent_event_stream_ordering joined_room_stream_ordering_update.most_recent_event_stream_ordering
) )
@ -1876,6 +1870,8 @@ class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseS
+ "Raising exception so we can just try again." + "Raising exception so we can just try again."
) )
# Handle updating the `sliding_sync_joined_rooms` table
#
# Since we fully insert rows into `sliding_sync_joined_rooms`, we can # Since we fully insert rows into `sliding_sync_joined_rooms`, we can
# just do everything on insert and `ON CONFLICT DO NOTHING`. # just do everything on insert and `ON CONFLICT DO NOTHING`.
# #
@ -1894,21 +1890,27 @@ class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseS
}, },
) )
# Now that we've processed all the room, we can remove them from the async def fill_table_for_room(room_id: str) -> None:
await self.db_pool.runInteraction(
"sliding_sync_joined_rooms_bg_update", _fill_table_for_room_txn, room_id
)
# Since most of the time spent here is probably just latency to/from the
# database, let's just run this concurrently.
await concurrently_execute(fill_table_for_room, joined_room_updates.keys(), 10)
# Now that we've processed all of the rooms, we can remove them from the
# queue. # queue.
# #
# Note: we need to remove all the rooms from the queue we pulled out # Note: we need to remove all the rooms from the queue we pulled out
# from the DB, not just the ones we've processed above. Otherwise # from the DB, not just the ones we've processed above. Otherwise
# we'll simply keep pulling out the same rooms over and over again. # we'll simply keep pulling out the same rooms over and over again.
self.db_pool.simple_delete_many_batch_txn( await self.db_pool.simple_delete_many(
txn,
table="sliding_sync_joined_rooms_to_recalculate", table="sliding_sync_joined_rooms_to_recalculate",
keys=("room_id",), column="room_id",
values=rooms_to_update_rows, iterable=rooms_to_update,
) keyvalues={},
desc="_sliding_sync_joined_rooms_bg_update: removing rooms that we processed",
await self.db_pool.runInteraction(
"sliding_sync_joined_rooms_bg_update", _fill_table_txn
) )
return len(rooms_to_update_rows) return len(rooms_to_update_rows)