sliding-sync: spider all rooms on the user's account for search (#9514)

* sliding-sync: spider all rooms on the user's account for search

On startup, slowly accumulate room metadata for all rooms on the
user's account. This is so we can populate the local search cache
with enough data for it to function, obviating the need to have
separate code paths for sliding sync searches.

This will allow spotlight search to work with slow/no network
connectivity, though clicking on the room will still require a
round trip.

This is an explicit request from @ara4n to improve the snapiness
of room searches, despite the unbounded bandwidth costs requesting
all N rooms on the user's account.

* Comments and tweak defaults

* Review comments; remove SS search code

* bugfix: use setListRanges once the list has been set up

If we don't, then we send needless extra data and can cause
bugs because setList will wipe the index->room_id map, which
trips up SlidingRoomListStore.

src/MatrixClientPeg.ts

@@ -249,6 +249,7 @@ class MatrixClientPegClass implements IMatrixClientPeg {
                 this.matrixClient,
                 proxyUrl || this.matrixClient.baseUrl,
             );
+            SlidingSyncManager.instance.startSpidering(100, 50); // 100 rooms at a time, 50ms apart
         }
 
         // Connect the matrix client to the dispatcher and setting handlers

src/SlidingSyncManager.ts

@@ -52,7 +52,7 @@ import {
     SlidingSync,
 } from 'matrix-js-sdk/src/sliding-sync';
 import { logger } from "matrix-js-sdk/src/logger";
-import { IDeferred, defer } from 'matrix-js-sdk/src/utils';
+import { IDeferred, defer, sleep } from 'matrix-js-sdk/src/utils';
 
 // how long to long poll for
 const SLIDING_SYNC_TIMEOUT_MS = 20 * 1000;
@@ -109,6 +109,9 @@ export class SlidingSyncManager {
     public configure(client: MatrixClient, proxyUrl: string): SlidingSync {
         this.client = client;
         this.listIdToIndex = {};
+        DEFAULT_ROOM_SUBSCRIPTION_INFO.include_old_rooms.required_state.push(
+            [EventType.RoomMember, client.getUserId()],
+        );
         this.slidingSync = new SlidingSync(
             proxyUrl, [], DEFAULT_ROOM_SUBSCRIPTION_INFO, client, SLIDING_SYNC_TIMEOUT_MS,
         );
@@ -262,4 +265,60 @@ export class SlidingSyncManager {
         }
         return roomId;
     }
+
+    /**
+     * Retrieve all rooms on the user's account. Used for pre-populating the local search cache.
+     * Retrieval is gradual over time.
+     * @param batchSize The number of rooms to return in each request.
+     * @param gapBetweenRequestsMs The number of milliseconds to wait between requests.
+     */
+    public async startSpidering(batchSize: number, gapBetweenRequestsMs: number) {
+        await sleep(gapBetweenRequestsMs); // wait a bit as this is called on first render so let's let things load
+        const listIndex = this.getOrAllocateListIndex(SlidingSyncManager.ListSearch);
+        let startIndex = batchSize;
+        let hasMore = true;
+        let firstTime = true;
+        while (hasMore) {
+            const endIndex = startIndex + batchSize-1;
+            try {
+                const ranges = [[0, batchSize-1], [startIndex, endIndex]];
+                if (firstTime) {
+                    await this.slidingSync.setList(listIndex, {
+                        // e.g [0,19] [20,39] then [0,19] [40,59]. We keep [0,20] constantly to ensure
+                        // any changes to the list whilst spidering are caught.
+                        ranges: ranges,
+                        sort: [
+                            "by_recency", // this list isn't shown on the UI so just sorting by timestamp is enough
+                        ],
+                        timeline_limit: 0, // we only care about the room details, not messages in the room
+                        required_state: [
+                            [EventType.RoomJoinRules, ""], // the public icon on the room list
+                            [EventType.RoomAvatar, ""], // any room avatar
+                            [EventType.RoomTombstone, ""], // lets JS SDK hide rooms which are dead
+                            [EventType.RoomEncryption, ""], // lets rooms be configured for E2EE correctly
+                            [EventType.RoomCreate, ""], // for isSpaceRoom checks
+                            [EventType.RoomMember, this.client.getUserId()!], // lets the client calculate that we are in fact in the room
+                        ],
+                        // we don't include_old_rooms here in an effort to reduce the impact of spidering all rooms
+                        // on the user's account. This means some data in the search dialog results may be inaccurate
+                        // e.g membership of space, but this will be corrected when the user clicks on the room
+                        // as the direct room subscription does include old room iterations.
+                        filters: { // we get spaces via a different list, so filter them out
+                            not_room_types: ["m.space"],
+                        },
+                    });
+                } else {
+                    await this.slidingSync.setListRanges(listIndex, ranges);
+                }
+                // gradually request more over time
+                await sleep(gapBetweenRequestsMs);
+            } catch (err) {
+                // do nothing, as we reject only when we get interrupted but that's fine as the next
+                // request will include our data
+            }
+            hasMore = (endIndex+1) < this.slidingSync.getListData(listIndex)?.joinedCount;
+            startIndex += batchSize;
+            firstTime = false;
+        }
+    }
 }

src/components/views/dialogs/spotlight/SpotlightDialog.tsx

@@ -91,7 +91,6 @@ import { RoomResultContextMenus } from "./RoomResultContextMenus";
 import { RoomContextDetails } from "../../rooms/RoomContextDetails";
 import { TooltipOption } from "./TooltipOption";
 import { isLocalRoom } from "../../../../utils/localRoom/isLocalRoom";
-import { useSlidingSyncRoomSearch } from "../../../../hooks/useSlidingSyncRoomSearch";
 import { shouldShowFeedback } from "../../../../utils/Feedback";
 import RoomAvatar from "../../avatars/RoomAvatar";
 
@@ -342,43 +341,26 @@ const SpotlightDialog: React.FC<IProps> = ({ initialText = "", initialFilter = n
         searchProfileInfo,
         searchParams,
     );
-    const isSlidingSyncEnabled = SettingsStore.getValue("feature_sliding_sync");
-    let {
-        loading: slidingSyncRoomSearchLoading,
-        rooms: slidingSyncRooms,
-        search: searchRoomsServerside,
-    } = useSlidingSyncRoomSearch();
-    useDebouncedCallback(isSlidingSyncEnabled, searchRoomsServerside, searchParams);
-    if (!isSlidingSyncEnabled) {
-        slidingSyncRoomSearchLoading = false;
-    }
 
     const possibleResults = useMemo<Result[]>(
         () => {
             const userResults: IMemberResult[] = [];
-            let roomResults: IRoomResult[];
-            let alreadyAddedUserIds: Set<string>;
-            if (isSlidingSyncEnabled) {
-                // use the rooms sliding sync returned as the server has already worked it out for us
-                roomResults = slidingSyncRooms.map(toRoomResult);
-            } else {
-                roomResults = findVisibleRooms(cli).map(toRoomResult);
-                // If we already have a DM with the user we're looking for, we will
-                // show that DM instead of the user themselves
-                alreadyAddedUserIds = roomResults.reduce((userIds, result) => {
-                    const userId = DMRoomMap.shared().getUserIdForRoomId(result.room.roomId);
-                    if (!userId) return userIds;
-                    if (result.room.getJoinedMemberCount() > 2) return userIds;
-                    userIds.add(userId);
-                    return userIds;
-                }, new Set<string>());
-                for (const user of [...findVisibleRoomMembers(cli), ...users]) {
-                    // Make sure we don't have any user more than once
-                    if (alreadyAddedUserIds.has(user.userId)) continue;
-                    alreadyAddedUserIds.add(user.userId);
+            const roomResults = findVisibleRooms(cli).map(toRoomResult);
+            // If we already have a DM with the user we're looking for, we will
+            // show that DM instead of the user themselves
+            const alreadyAddedUserIds = roomResults.reduce((userIds, result) => {
+                const userId = DMRoomMap.shared().getUserIdForRoomId(result.room.roomId);
+                if (!userId) return userIds;
+                if (result.room.getJoinedMemberCount() > 2) return userIds;
+                userIds.add(userId);
+                return userIds;
+            }, new Set<string>());
+            for (const user of [...findVisibleRoomMembers(cli), ...users]) {
+                // Make sure we don't have any user more than once
+                if (alreadyAddedUserIds.has(user.userId)) continue;
+                alreadyAddedUserIds.add(user.userId);
 
-                    userResults.push(toMemberResult(user));
-                }
+                userResults.push(toMemberResult(user));
             }
 
             return [
@@ -402,7 +384,7 @@ const SpotlightDialog: React.FC<IProps> = ({ initialText = "", initialFilter = n
                 ...publicRooms.map(toPublicRoomResult),
             ].filter(result => filter === null || result.filter.includes(filter));
         },
-        [cli, users, profile, publicRooms, slidingSyncRooms, isSlidingSyncEnabled, filter],
+        [cli, users, profile, publicRooms, filter],
     );
 
     const results = useMemo<Record<Section, Result[]>>(() => {
@@ -421,13 +403,10 @@ const SpotlightDialog: React.FC<IProps> = ({ initialText = "", initialFilter = n
 
             possibleResults.forEach(entry => {
                 if (isRoomResult(entry)) {
-                    // sliding sync gives the correct rooms in the list so we don't need to filter
-                    if (!isSlidingSyncEnabled) {
-                        if (!entry.room.normalizedName?.includes(normalizedQuery) &&
-                            !entry.room.getCanonicalAlias()?.toLowerCase().includes(lcQuery) &&
-                            !entry.query?.some(q => q.includes(lcQuery))
-                        ) return; // bail, does not match query
-                    }
+                    if (!entry.room.normalizedName?.includes(normalizedQuery) &&
+                        !entry.room.getCanonicalAlias()?.toLowerCase().includes(lcQuery) &&
+                        !entry.query?.some(q => q.includes(lcQuery))
+                    ) return; // bail, does not match query
                 } else if (isMemberResult(entry)) {
                     if (!entry.query?.some(q => q.includes(lcQuery))) return; // bail, does not match query
                 } else if (isPublicRoomResult(entry)) {
@@ -478,7 +457,7 @@ const SpotlightDialog: React.FC<IProps> = ({ initialText = "", initialFilter = n
         }
 
         return results;
-    }, [trimmedQuery, filter, cli, possibleResults, isSlidingSyncEnabled, memberComparator]);
+    }, [trimmedQuery, filter, cli, possibleResults, memberComparator]);
 
     const numResults = sum(Object.values(results).map(it => it.length));
     useWebSearchMetrics(numResults, query.length, true);
@@ -1236,7 +1215,7 @@ const SpotlightDialog: React.FC<IProps> = ({ initialText = "", initialFilter = n
                     aria-label={_t("Search")}
                     aria-describedby="mx_SpotlightDialog_keyboardPrompt"
                 />
-                { (publicRoomsLoading || peopleLoading || profileLoading || slidingSyncRoomSearchLoading) && (
+                { (publicRoomsLoading || peopleLoading || profileLoading) && (
                     <Spinner w={24} h={24} />
                 ) }
             </div>

test/SlidingSyncManager-test.ts

@@ -0,0 +1,114 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { SlidingSync } from 'matrix-js-sdk/src/sliding-sync';
+import { mocked } from 'jest-mock';
+
+import { SlidingSyncManager } from '../src/SlidingSyncManager';
+import { stubClient } from './test-utils';
+
+jest.mock('matrix-js-sdk/src/sliding-sync');
+const MockSlidingSync = <jest.Mock<SlidingSync>><unknown>SlidingSync;
+
+describe('SlidingSyncManager', () => {
+    let manager: SlidingSyncManager;
+    let slidingSync: SlidingSync;
+
+    beforeEach(() => {
+        slidingSync = new MockSlidingSync();
+        manager = new SlidingSyncManager();
+        manager.configure(stubClient(), "invalid");
+        manager.slidingSync = slidingSync;
+    });
+
+    describe("startSpidering", () => {
+        it("requests in batchSizes", async () => {
+            const gapMs = 1;
+            const batchSize = 10;
+            mocked(slidingSync.setList).mockResolvedValue("yep");
+            mocked(slidingSync.setListRanges).mockResolvedValue("yep");
+            mocked(slidingSync.getListData).mockImplementation((i) => {
+                return {
+                    joinedCount: 64,
+                    roomIndexToRoomId: {},
+                };
+            });
+            await manager.startSpidering(batchSize, gapMs);
+            // we expect calls for 10,19 -> 20,29 -> 30,39 -> 40,49 -> 50,59 -> 60,69
+            const wantWindows = [
+                [10, 19], [20, 29], [30, 39], [40, 49], [50, 59], [60, 69],
+            ];
+            expect(slidingSync.getListData).toBeCalledTimes(wantWindows.length);
+            expect(slidingSync.setList).toBeCalledTimes(1);
+            expect(slidingSync.setListRanges).toBeCalledTimes(wantWindows.length-1);
+            wantWindows.forEach((range, i) => {
+                if (i === 0) {
+                    expect(slidingSync.setList).toBeCalledWith(
+                        manager.getOrAllocateListIndex(SlidingSyncManager.ListSearch),
+                        expect.objectContaining({
+                            ranges: [[0, batchSize-1], range],
+                        }),
+                    );
+                    return;
+                }
+                expect(slidingSync.setListRanges).toBeCalledWith(
+                    manager.getOrAllocateListIndex(SlidingSyncManager.ListSearch),
+                    [[0, batchSize-1], range],
+                );
+            });
+        });
+        it("handles accounts with zero rooms", async () => {
+            const gapMs = 1;
+            const batchSize = 10;
+            mocked(slidingSync.setList).mockResolvedValue("yep");
+            mocked(slidingSync.getListData).mockImplementation((i) => {
+                return {
+                    joinedCount: 0,
+                    roomIndexToRoomId: {},
+                };
+            });
+            await manager.startSpidering(batchSize, gapMs);
+            expect(slidingSync.getListData).toBeCalledTimes(1);
+            expect(slidingSync.setList).toBeCalledTimes(1);
+            expect(slidingSync.setList).toBeCalledWith(
+                manager.getOrAllocateListIndex(SlidingSyncManager.ListSearch),
+                expect.objectContaining({
+                    ranges: [[0, batchSize-1], [batchSize, batchSize+batchSize-1]],
+                }),
+            );
+        });
+        it("continues even when setList rejects", async () => {
+            const gapMs = 1;
+            const batchSize = 10;
+            mocked(slidingSync.setList).mockRejectedValue("narp");
+            mocked(slidingSync.getListData).mockImplementation((i) => {
+                return {
+                    joinedCount: 0,
+                    roomIndexToRoomId: {},
+                };
+            });
+            await manager.startSpidering(batchSize, gapMs);
+            expect(slidingSync.getListData).toBeCalledTimes(1);
+            expect(slidingSync.setList).toBeCalledTimes(1);
+            expect(slidingSync.setList).toBeCalledWith(
+                manager.getOrAllocateListIndex(SlidingSyncManager.ListSearch),
+                expect.objectContaining({
+                    ranges: [[0, batchSize-1], [batchSize, batchSize+batchSize-1]],
+                }),
+            );
+        });
+    });
+});