element-web/src/SlidingSyncManager.ts

/*
Copyright 2024 New Vector Ltd.
Copyright 2022 The Matrix.org Foundation C.I.C.

SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only
Please see LICENSE files in the repository root for full details.
*/

/*
 * Sliding Sync Architecture - MSC https://github.com/matrix-org/matrix-spec-proposals/pull/3575
 *
 * This is a holistic summary of the changes made to Element-Web / React SDK / JS SDK to enable sliding sync.
 * This summary will hopefully signpost where developers need to look if they want to make changes to this code.
 *
 * At the lowest level, the JS SDK contains an HTTP API wrapper function in client.ts. This is used by
 * a SlidingSync class in JS SDK, which contains code to handle list operations (INSERT/DELETE/SYNC/etc)
 * and contains the main request API bodies, but has no code to control updating JS SDK structures: it just
 * exposes an EventEmitter to listen for updates. When MatrixClient.startClient is called, callers need to
 * provide a SlidingSync instance as this contains the main request API params (timeline limit, required state,
 * how many lists, etc).
 *
 * The SlidingSyncSdk INTERNAL class in JS SDK attaches listeners to SlidingSync to update JS SDK Room objects,
 * and it conveniently exposes an identical public API to SyncApi (to allow it to be a drop-in replacement).
 *
 * At the highest level, SlidingSyncManager contains mechanisms to tell UI lists which rooms to show,
 * and contains the core request API params used in Element-Web. It does this by listening for events
 * emitted by the SlidingSync class and by modifying the request API params on the SlidingSync class.
 *
 *    (entry point)                     (updates JS SDK)
 *  SlidingSyncManager                   SlidingSyncSdk
 *       |                                     |
 *       +------------------.------------------+
 *         listens          |          listens
 *                     SlidingSync
 *                     (sync loop,
 *                      list ops)
 */

import { MatrixClient, EventType, AutoDiscovery, Method, timeoutSignal } from "matrix-js-sdk/src/matrix";
import {
    MSC3575Filter,
    MSC3575List,
    MSC3575_STATE_KEY_LAZY,
    MSC3575_STATE_KEY_ME,
    MSC3575_WILDCARD,
    SlidingSync,
} from "matrix-js-sdk/src/sliding-sync";
import { logger } from "matrix-js-sdk/src/logger";
import { defer, sleep } from "matrix-js-sdk/src/utils";

import SettingsStore from "./settings/SettingsStore";
import SlidingSyncController from "./settings/controllers/SlidingSyncController";

// how long to long poll for
const SLIDING_SYNC_TIMEOUT_MS = 20 * 1000;

// the things to fetch when a user clicks on a room
const DEFAULT_ROOM_SUBSCRIPTION_INFO = {
    timeline_limit: 50,
    // missing required_state which will change depending on the kind of room
    include_old_rooms: {
        timeline_limit: 0,
        required_state: [
            // state needed to handle space navigation and tombstone chains
            [EventType.RoomCreate, ""],
            [EventType.RoomTombstone, ""],
            [EventType.SpaceChild, MSC3575_WILDCARD],
            [EventType.SpaceParent, MSC3575_WILDCARD],
            [EventType.RoomMember, MSC3575_STATE_KEY_ME],
        ],
    },
};
// lazy load room members so rooms like Matrix HQ don't take forever to load
const UNENCRYPTED_SUBSCRIPTION_NAME = "unencrypted";
const UNENCRYPTED_SUBSCRIPTION = {
    required_state: [
        [MSC3575_WILDCARD, MSC3575_WILDCARD], // all events
        [EventType.RoomMember, MSC3575_STATE_KEY_ME], // except for m.room.members, get our own membership
        [EventType.RoomMember, MSC3575_STATE_KEY_LAZY], // ...and lazy load the rest.
    ],
    ...DEFAULT_ROOM_SUBSCRIPTION_INFO,
};

// we need all the room members in encrypted rooms because we need to know which users to encrypt
// messages for.
const ENCRYPTED_SUBSCRIPTION = {
    required_state: [
        [MSC3575_WILDCARD, MSC3575_WILDCARD], // all events
    ],
    ...DEFAULT_ROOM_SUBSCRIPTION_INFO,
};

export type PartialSlidingSyncRequest = {
    filters?: MSC3575Filter;
    sort?: string[];
    ranges?: [startIndex: number, endIndex: number][];
};

/**
 * This class manages the entirety of sliding sync at a high UI/UX level. It controls the placement
 * of placeholders in lists, controls updating sliding window ranges, and controls which events
 * are pulled down when. The intention behind this manager is be the single place to look for sliding
 * sync options and code.
 */
export class SlidingSyncManager {
    public static readonly ListSpaces = "space_list";
    public static readonly ListSearch = "search_list";
    private static readonly internalInstance = new SlidingSyncManager();

    public slidingSync?: SlidingSync;
    private client?: MatrixClient;

    private configureDefer = defer<void>();

    public static get instance(): SlidingSyncManager {
        return SlidingSyncManager.internalInstance;
    }

    public configure(client: MatrixClient, proxyUrl: string): SlidingSync {
        this.client = client;
        // by default use the encrypted subscription as that gets everything, which is a safer
        // default than potentially missing member events.
        this.slidingSync = new SlidingSync(
            proxyUrl,
            new Map(),
            ENCRYPTED_SUBSCRIPTION,
            client,
            SLIDING_SYNC_TIMEOUT_MS,
        );
        this.slidingSync.addCustomSubscription(UNENCRYPTED_SUBSCRIPTION_NAME, UNENCRYPTED_SUBSCRIPTION);
        // set the space list
        this.slidingSync.setList(SlidingSyncManager.ListSpaces, {
            ranges: [[0, 20]],
            sort: ["by_name"],
            slow_get_all_rooms: true,
            timeline_limit: 0,
            required_state: [
                [EventType.RoomJoinRules, ""], // the public icon on the room list
                [EventType.RoomAvatar, ""], // any room avatar
                [EventType.RoomTombstone, ""], // lets JS SDK hide rooms which are dead
                [EventType.RoomEncryption, ""], // lets rooms be configured for E2EE correctly
                [EventType.RoomCreate, ""], // for isSpaceRoom checks
                [EventType.SpaceChild, MSC3575_WILDCARD], // all space children
                [EventType.SpaceParent, MSC3575_WILDCARD], // all space parents
                [EventType.RoomMember, MSC3575_STATE_KEY_ME], // lets the client calculate that we are in fact in the room
            ],
            include_old_rooms: {
                timeline_limit: 0,
                required_state: [
                    [EventType.RoomCreate, ""],
                    [EventType.RoomTombstone, ""], // lets JS SDK hide rooms which are dead
                    [EventType.SpaceChild, MSC3575_WILDCARD], // all space children
                    [EventType.SpaceParent, MSC3575_WILDCARD], // all space parents
                    [EventType.RoomMember, MSC3575_STATE_KEY_ME], // lets the client calculate that we are in fact in the room
                ],
            },
            filters: {
                room_types: ["m.space"],
            },
        });
        this.configureDefer.resolve();
        return this.slidingSync;
    }

    /**
     * Ensure that this list is registered.
     * @param listKey The list key to register
     * @param updateArgs The fields to update on the list.
     * @returns The complete list request params
     */
    public async ensureListRegistered(listKey: string, updateArgs: PartialSlidingSyncRequest): Promise<MSC3575List> {
        logger.debug("ensureListRegistered:::", listKey, updateArgs);
        await this.configureDefer.promise;
        let list = this.slidingSync!.getListParams(listKey);
        if (!list) {
            list = {
                ranges: [[0, 20]],
                sort: ["by_notification_level", "by_recency"],
                timeline_limit: 1, // most recent message display: though this seems to only be needed for favourites?
                required_state: [
                    [EventType.RoomJoinRules, ""], // the public icon on the room list
                    [EventType.RoomAvatar, ""], // any room avatar
                    [EventType.RoomTombstone, ""], // lets JS SDK hide rooms which are dead
                    [EventType.RoomEncryption, ""], // lets rooms be configured for E2EE correctly
                    [EventType.RoomCreate, ""], // for isSpaceRoom checks
                    [EventType.RoomMember, MSC3575_STATE_KEY_ME], // lets the client calculate that we are in fact in the room
                ],
                include_old_rooms: {
                    timeline_limit: 0,
                    required_state: [
                        [EventType.RoomCreate, ""],
                        [EventType.RoomTombstone, ""], // lets JS SDK hide rooms which are dead
                        [EventType.SpaceChild, MSC3575_WILDCARD], // all space children
                        [EventType.SpaceParent, MSC3575_WILDCARD], // all space parents
                        [EventType.RoomMember, MSC3575_STATE_KEY_ME], // lets the client calculate that we are in fact in the room
                    ],
                },
                ...updateArgs,
            };
        } else {
            const updatedList = { ...list, ...updateArgs };
            // cannot use objectHasDiff as we need to do deep diff checking
            if (JSON.stringify(list) === JSON.stringify(updatedList)) {
                logger.debug("list matches, not sending, update => ", updateArgs);
                return list;
            }
            list = updatedList;
        }

        try {
            // if we only have range changes then call a different function so we don't nuke the list from before
            if (updateArgs.ranges && Object.keys(updateArgs).length === 1) {
                await this.slidingSync!.setListRanges(listKey, updateArgs.ranges);
            } else {
                await this.slidingSync!.setList(listKey, list);
            }
        } catch (err) {
            logger.debug("ensureListRegistered: update failed txn_id=", err);
        }
        return this.slidingSync!.getListParams(listKey)!;
    }

    public async setRoomVisible(roomId: string, visible: boolean): Promise<string> {
        await this.configureDefer.promise;
        const subscriptions = this.slidingSync!.getRoomSubscriptions();
        if (visible) {
            subscriptions.add(roomId);
        } else {
            subscriptions.delete(roomId);
        }
        const room = this.client?.getRoom(roomId);
        let shouldLazyLoad = !this.client?.isRoomEncrypted(roomId);
        if (!room) {
            // default to safety: request all state if we can't work it out. This can happen if you
            // refresh the app whilst viewing a room: we call setRoomVisible before we know anything
            // about the room.
            shouldLazyLoad = false;
        }
        logger.log("SlidingSync setRoomVisible:", roomId, visible, "shouldLazyLoad:", shouldLazyLoad);
        if (shouldLazyLoad) {
            // lazy load this room
            this.slidingSync!.useCustomSubscription(roomId, UNENCRYPTED_SUBSCRIPTION_NAME);
        }
        const p = this.slidingSync!.modifyRoomSubscriptions(subscriptions);
        if (room) {
            return roomId; // we have data already for this room, show immediately e.g it's in a list
        }
        try {
            // wait until the next sync before returning as RoomView may need to know the current state
            await p;
        } catch {
            logger.warn("SlidingSync setRoomVisible:", roomId, visible, "failed to confirm transaction");
        }
        return roomId;
    }

    /**
     * Retrieve all rooms on the user's account. Used for pre-populating the local search cache.
     * Retrieval is gradual over time.
     * @param batchSize The number of rooms to return in each request.
     * @param gapBetweenRequestsMs The number of milliseconds to wait between requests.
     */
    public async startSpidering(batchSize: number, gapBetweenRequestsMs: number): Promise<void> {
        await sleep(gapBetweenRequestsMs); // wait a bit as this is called on first render so let's let things load
        let startIndex = batchSize;
        let hasMore = true;
        let firstTime = true;
        while (hasMore) {
            const endIndex = startIndex + batchSize - 1;
            try {
                const ranges = [
                    [0, batchSize - 1],
                    [startIndex, endIndex],
                ];
                if (firstTime) {
                    await this.slidingSync!.setList(SlidingSyncManager.ListSearch, {
                        // e.g [0,19] [20,39] then [0,19] [40,59]. We keep [0,20] constantly to ensure
                        // any changes to the list whilst spidering are caught.
                        ranges: ranges,
                        sort: [
                            "by_recency", // this list isn't shown on the UI so just sorting by timestamp is enough
                        ],
                        timeline_limit: 0, // we only care about the room details, not messages in the room
                        required_state: [
                            [EventType.RoomJoinRules, ""], // the public icon on the room list
                            [EventType.RoomAvatar, ""], // any room avatar
                            [EventType.RoomTombstone, ""], // lets JS SDK hide rooms which are dead
                            [EventType.RoomEncryption, ""], // lets rooms be configured for E2EE correctly
                            [EventType.RoomCreate, ""], // for isSpaceRoom checks
                            [EventType.RoomMember, MSC3575_STATE_KEY_ME], // lets the client calculate that we are in fact in the room
                        ],
                        // we don't include_old_rooms here in an effort to reduce the impact of spidering all rooms
                        // on the user's account. This means some data in the search dialog results may be inaccurate
                        // e.g membership of space, but this will be corrected when the user clicks on the room
                        // as the direct room subscription does include old room iterations.
                        filters: {
                            // we get spaces via a different list, so filter them out
                            not_room_types: ["m.space"],
                        },
                    });
                } else {
                    await this.slidingSync!.setListRanges(SlidingSyncManager.ListSearch, ranges);
                }
            } catch {
                // do nothing, as we reject only when we get interrupted but that's fine as the next
                // request will include our data
            } finally {
                // gradually request more over time, even on errors.
                await sleep(gapBetweenRequestsMs);
            }
            const listData = this.slidingSync!.getListData(SlidingSyncManager.ListSearch)!;
            hasMore = endIndex + 1 < listData.joinedCount;
            startIndex += batchSize;
            firstTime = false;
        }
    }

    /**
     * Set up the Sliding Sync instance; configures the end point and starts spidering.
     * The sliding sync endpoint is derived the following way:
     *   1. The user-defined sliding sync proxy URL (legacy, for backwards compatibility)
     *   2. The client `well-known` sliding sync proxy URL [declared at the unstable prefix](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/sync-v3/proposals/3575-sync.md#unstable-prefix)
     *   3. The homeserver base url (for native server support)
     * @param client The MatrixClient to use
     * @returns A working Sliding Sync or undefined
     */
    public async setup(client: MatrixClient): Promise<SlidingSync | undefined> {
        const baseUrl = client.baseUrl;
        const proxyUrl = SettingsStore.getValue("feature_sliding_sync_proxy_url");
        const wellKnownProxyUrl = await this.getProxyFromWellKnown(client);

        const slidingSyncEndpoint = proxyUrl || wellKnownProxyUrl || baseUrl;

        this.configure(client, slidingSyncEndpoint);
        logger.info("Sliding sync activated at", slidingSyncEndpoint);
        this.startSpidering(100, 50); // 100 rooms at a time, 50ms apart

        return this.slidingSync;
    }

    /**
     * Get the sliding sync proxy URL from the client well known
     * @param client The MatrixClient to use
     * @return The proxy url
     */
    public async getProxyFromWellKnown(client: MatrixClient): Promise<string | undefined> {
        let proxyUrl: string | undefined;

        try {
            const clientDomain = await client.getDomain();
            if (clientDomain === null) {
                throw new RangeError("Homeserver domain is null");
            }
            const clientWellKnown = await AutoDiscovery.findClientConfig(clientDomain);
            proxyUrl = clientWellKnown?.["org.matrix.msc3575.proxy"]?.url;
        } catch {
            // Either client.getDomain() is null so we've shorted out, or is invalid so `AutoDiscovery.findClientConfig` has thrown
        }

        if (proxyUrl != undefined) {
            logger.log("getProxyFromWellKnown: client well-known declares sliding sync proxy at", proxyUrl);
        }
        return proxyUrl;
    }

    /**
     * Check if the server "natively" supports sliding sync (with an unstable endpoint).
     * @param client The MatrixClient to use
     * @return Whether the "native" (unstable) endpoint is supported
     */
    public async nativeSlidingSyncSupport(client: MatrixClient): Promise<boolean> {
        // Per https://github.com/matrix-org/matrix-spec-proposals/pull/3575/files#r1589542561
        // `client` can be undefined/null in tests for some reason.
        const support = await client?.doesServerSupportUnstableFeature("org.matrix.msc3575");
        if (support) {
            logger.log("nativeSlidingSyncSupport: sliding sync advertised as unstable");
        }
        return support;
    }

    /**
     * Check whether our homeserver has sliding sync support, that the endpoint is up, and
     * is a sliding sync endpoint.
     *
     * Sets static member `SlidingSyncController.serverSupportsSlidingSync`
     * @param client The MatrixClient to use
     */
    public async checkSupport(client: MatrixClient): Promise<void> {
        if (await this.nativeSlidingSyncSupport(client)) {
            SlidingSyncController.serverSupportsSlidingSync = true;
            return;
        }

        const proxyUrl = await this.getProxyFromWellKnown(client);
        if (proxyUrl != undefined) {
            const response = await fetch(new URL("/client/server.json", proxyUrl), {
                method: Method.Get,
                signal: timeoutSignal(10 * 1000), // 10s
            });
            if (response.status === 200) {
                logger.log("checkSupport: well-known sliding sync proxy is up at", proxyUrl);
                SlidingSyncController.serverSupportsSlidingSync = true;
            }
        }
    }
}