Show a progress bar while migrating from legacy crypto (#12104)

* Show a progress bar during migration of crypto data

* playwright: add new `pageWithCredentials` fixture

* Add a playwright test for migration progress

* Add documentation for `idbSave`

.prettierignore

@@ -17,3 +17,6 @@ yarn.lock
 
 # This file is owned, parsed, and generated by allchange, which doesn't comply with prettier
 /CHANGELOG.md
+
+# This file is also machine-generated
+/playwright/e2e/crypto/test_indexeddb_cryptostore_dump/dump.json

playwright/e2e/crypto/migration.spec.ts

@@ -0,0 +1,72 @@
+/*
+Copyright 2023-2024 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import path from "path";
+import { readFile } from "node:fs/promises";
+
+import { expect, test as base } from "../../element-web-test";
+
+const test = base.extend({
+    // Replace the `user` fixture with one which populates the indexeddb data before starting the app.
+    user: async ({ context, pageWithCredentials: page, credentials }, use) => {
+        await page.route(`/test_indexeddb_cryptostore_dump/*`, async (route, request) => {
+            const resourcePath = path.join(__dirname, new URL(request.url()).pathname);
+            const body = await readFile(resourcePath, { encoding: "utf-8" });
+            await route.fulfill({ body });
+        });
+        await page.goto("/test_indexeddb_cryptostore_dump/index.html");
+
+        await use(credentials);
+    },
+});
+
+test.describe("migration", function () {
+    test.use({ displayName: "Alice" });
+
+    test("Should support migration from legacy crypto", async ({ context, user, page }, workerInfo) => {
+        test.skip(workerInfo.project.name === "Legacy Crypto", "This test only works with Rust crypto.");
+        test.slow();
+
+        // We should see a migration progress bar
+        await page.getByText("Hang tight.").waitFor({ timeout: 60000 });
+
+        // When the progress bar first loads, it should have a high max (one per megolm session to import), and
+        // a relatively low value.
+        const progressBar = page.getByRole("progressbar");
+        const initialProgress = parseFloat(await progressBar.getAttribute("value"));
+        const initialMax = parseFloat(await progressBar.getAttribute("max"));
+        expect(initialMax).toBeGreaterThan(4000);
+        expect(initialProgress).toBeGreaterThanOrEqual(0);
+        expect(initialProgress).toBeLessThanOrEqual(500);
+
+        // Later, the progress should pass 50%
+        await expect
+            .poll(
+                async () => {
+                    const progressBar = page.getByRole("progressbar");
+                    return (
+                        (parseFloat(await progressBar.getAttribute("value")) * 100.0) /
+                        parseFloat(await progressBar.getAttribute("max"))
+                    );
+                },
+                { timeout: 60000 },
+            )
+            .toBeGreaterThan(50);
+
+        // Eventually, we should get a normal matrix chat
+        await page.waitForSelector(".mx_MatrixChat", { timeout: 120000 });
+    });
+});

playwright/e2e/crypto/test_indexeddb_cryptostore_dump/README.md

@@ -0,0 +1,60 @@
+# Dump of libolm indexeddb cryptostore
+
+This directory contains, in `dump.json`, a dump of a real indexeddb store from a session using
+libolm crypto.
+
+The corresponding pickle key is `+1k2Ppd7HIisUY824v7JtV3/oEE4yX0TqtmNPyhaD7o`.
+
+This directory also contains, in `index.html` and `load.js`, a page which will populate indexeddb with the data
+(and the pickle key). This can be served via a Playwright [Route](https://playwright.dev/docs/api/class-route) so as to
+populate the indexeddb before the main application loads. Note that encrypting the pickle key requires the test User ID
+and Device ID, so they must be stored in `localstorage` before loading `index.html`.
+
+## Creation of the dump file
+
+The dump was created by pasting the following into the browser console:
+
+```javascript
+async function exportIndexedDb(name) {
+    const db = await new Promise((resolve, reject) => {
+        const dbReq = indexedDB.open(name);
+        dbReq.onerror = reject;
+        dbReq.onsuccess = () => resolve(dbReq.result);
+    });
+
+    const storeNames = db.objectStoreNames;
+    const exports = {};
+    for (const store of storeNames) {
+        exports[store] = [];
+        const txn = db.transaction(store, "readonly");
+        const objectStore = txn.objectStore(store);
+        await new Promise((resolve, reject) => {
+            const cursorReq = objectStore.openCursor();
+            cursorReq.onerror = reject;
+            cursorReq.onsuccess = (event) => {
+                const cursor = event.target.result;
+                if (cursor) {
+                    const entry = { value: cursor.value };
+                    if (!objectStore.keyPath) {
+                        entry.key = cursor.key;
+                    }
+                    exports[store].push(entry);
+                    cursor.continue();
+                } else {
+                    resolve();
+                }
+            };
+        });
+    }
+    return exports;
+}
+
+window.saveAs(
+    new Blob([JSON.stringify(await exportIndexedDb("matrix-js-sdk:crypto"), null, 2)], {
+        type: "application/json;charset=utf-8",
+    }),
+    "dump.json",
+);
+```
+
+The pickle key is extracted via `mxMatrixClientPeg.get().crypto.olmDevice.pickleKey`.

playwright/e2e/crypto/test_indexeddb_cryptostore_dump/index.html

@@ -0,0 +1,6 @@
+<html>
+    <head>
+        <script src="load.js"></script>
+    </head>
+    Loading test data...
+</html>

playwright/e2e/crypto/test_indexeddb_cryptostore_dump/load.js

@@ -0,0 +1,228 @@
+/*
+Copyright 2023-2024 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/* Browser-side javascript to fetch the indexeddb dump file, and populate indexeddb. */
+
+/** The pickle key corresponding to the data dump. */
+const PICKLE_KEY = "+1k2Ppd7HIisUY824v7JtV3/oEE4yX0TqtmNPyhaD7o";
+
+/**
+ * Populate an IndexedDB store with the test data from this directory.
+ *
+ * @param {any} data - IndexedDB dump to import
+ * @param {string} name - Name of the IndexedDB database to create.
+ */
+async function populateStore(data, name) {
+    const req = indexedDB.open(name, 11);
+
+    const db = await new Promise((resolve, reject) => {
+        req.onupgradeneeded = (ev) => {
+            const db = req.result;
+            const oldVersion = ev.oldVersion;
+            upgradeDatabase(oldVersion, db);
+        };
+
+        req.onerror = (ev) => {
+            reject(req.error);
+        };
+
+        req.onsuccess = () => {
+            const db = req.result;
+            resolve(db);
+        };
+    });
+
+    await importData(data, db);
+
+    return db;
+}
+
+/**
+ * Create the schema for the indexed db store
+ *
+ * @param {number} oldVersion - The current version of the store.
+ * @param {IDBDatabase} db - The indexeddb database.
+ */
+function upgradeDatabase(oldVersion, db) {
+    if (oldVersion < 1) {
+        const outgoingRoomKeyRequestsStore = db.createObjectStore("outgoingRoomKeyRequests", { keyPath: "requestId" });
+        outgoingRoomKeyRequestsStore.createIndex("session", ["requestBody.room_id", "requestBody.session_id"]);
+        outgoingRoomKeyRequestsStore.createIndex("state", "state");
+    }
+
+    if (oldVersion < 2) {
+        db.createObjectStore("account");
+    }
+
+    if (oldVersion < 3) {
+        const sessionsStore = db.createObjectStore("sessions", { keyPath: ["deviceKey", "sessionId"] });
+        sessionsStore.createIndex("deviceKey", "deviceKey");
+    }
+
+    if (oldVersion < 4) {
+        db.createObjectStore("inbound_group_sessions", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 5) {
+        db.createObjectStore("device_data");
+    }
+
+    if (oldVersion < 6) {
+        db.createObjectStore("rooms");
+    }
+
+    if (oldVersion < 7) {
+        db.createObjectStore("sessions_needing_backup", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 8) {
+        db.createObjectStore("inbound_group_sessions_withheld", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 9) {
+        const problemsStore = db.createObjectStore("session_problems", { keyPath: ["deviceKey", "time"] });
+        problemsStore.createIndex("deviceKey", "deviceKey");
+
+        db.createObjectStore("notified_error_devices", { keyPath: ["userId", "deviceId"] });
+    }
+
+    if (oldVersion < 10) {
+        db.createObjectStore("shared_history_inbound_group_sessions", { keyPath: ["roomId"] });
+    }
+
+    if (oldVersion < 11) {
+        db.createObjectStore("parked_shared_history", { keyPath: ["roomId"] });
+    }
+}
+
+/** Do the import of data into the database
+ *
+ * @param {any} json - The data to import.
+ * @param {IDBDatabase} db - The database to import into.
+ * @returns {Promise<void>}
+ */
+async function importData(json, db) {
+    for (const [storeName, data] of Object.entries(json)) {
+        await new Promise((resolve, reject) => {
+            console.log(`Populating ${storeName} with test data`);
+            const store = db.transaction(storeName, "readwrite").objectStore(storeName);
+
+            function putEntry(idx) {
+                if (idx >= data.length) {
+                    resolve(undefined);
+                    return;
+                }
+
+                const { key, value } = data[idx];
+                try {
+                    const putReq = store.put(value, key);
+                    putReq.onsuccess = (_) => putEntry(idx + 1);
+                    putReq.onerror = (_) => reject(putReq.error);
+                } catch (e) {
+                    throw new Error(
+                        `Error populating '${storeName}' with key ${JSON.stringify(key)}, value ${JSON.stringify(
+                            value,
+                        )}: ${e}`,
+                    );
+                }
+            }
+
+            putEntry(0);
+        });
+    }
+}
+
+function getPickleAdditionalData(userId, deviceId) {
+    const additionalData = new Uint8Array(userId.length + deviceId.length + 1);
+    for (let i = 0; i < userId.length; i++) {
+        additionalData[i] = userId.charCodeAt(i);
+    }
+    additionalData[userId.length] = 124; // "|"
+    for (let i = 0; i < deviceId.length; i++) {
+        additionalData[userId.length + 1 + i] = deviceId.charCodeAt(i);
+    }
+    return additionalData;
+}
+
+/** Save an entry to the `matrix-react-sdk` indexeddb database.
+ *
+ * If `matrix-react-sdk` does not yet exist, it will be created with the correct schema.
+ *
+ * @param {String} table
+ * @param {String} key
+ * @param {String} data
+ * @returns {Promise<void>}
+ */
+async function idbSave(table, key, data) {
+    const idb = await new Promise((resolve, reject) => {
+        const request = indexedDB.open("matrix-react-sdk", 1);
+        request.onerror = reject;
+        request.onsuccess = () => {
+            resolve(request.result);
+        };
+        request.onupgradeneeded = () => {
+            const db = request.result;
+            db.createObjectStore("pickleKey");
+            db.createObjectStore("account");
+        };
+    });
+    return await new Promise((resolve, reject) => {
+        const txn = idb.transaction([table], "readwrite");
+        txn.onerror = reject;
+
+        const objectStore = txn.objectStore(table);
+        const request = objectStore.put(data, key);
+        request.onerror = reject;
+        request.onsuccess = resolve;
+    });
+}
+
+/**
+ * Save the pickle key to indexeddb, so that the app can read it.
+ *
+ * @param {String} userId  - The user's ID (used in the encryption algorithm).
+ * @param {String} deviceId  - The user's device ID (ditto).
+ * @returns {Promise<void>}
+ */
+async function savePickleKey(userId, deviceId) {
+    const itFunc = function* () {
+        const decoded = atob(PICKLE_KEY);
+        for (let i = 0; i < decoded.length; ++i) {
+            yield decoded.charCodeAt(i);
+        }
+    };
+    const decoded = Uint8Array.from(itFunc());
+
+    const cryptoKey = await crypto.subtle.generateKey({ name: "AES-GCM", length: 256 }, false, ["encrypt", "decrypt"]);
+    const iv = new Uint8Array(32);
+    crypto.getRandomValues(iv);
+
+    const additionalData = getPickleAdditionalData(userId, deviceId);
+    const encrypted = await crypto.subtle.encrypt({ name: "AES-GCM", iv, additionalData }, cryptoKey, decoded);
+
+    await idbSave("pickleKey", [userId, deviceId], { encrypted, iv, cryptoKey });
+}
+
+async function loadDump() {
+    const dump = await fetch("dump.json");
+    const indexedDbDump = await dump.json();
+    await populateStore(indexedDbDump, "matrix-js-sdk:crypto");
+    await savePickleKey(window.localStorage.getItem("mx_user_id"), window.localStorage.getItem("mx_device_id"));
+    console.log("Test data loaded; redirecting to main app");
+    window.location.replace("/");
+}
+
+loadDump();

playwright/element-web-test.ts

@@ -73,6 +73,16 @@ export const test = base.extend<
         homeserver: HomeserverInstance;
         oAuthServer: { port: number };
         credentials: CredentialsWithDisplayName;
+
+        /**
+         * The same as {@link https://playwright.dev/docs/api/class-fixtures#fixtures-page|`page`},
+         * but adds an initScript which will populate localStorage with the user's details from
+         * {@link #credentials} and {@link #homeserver}.
+         *
+         * Similar to {@link #user}, but doesn't load the app.
+         */
+        pageWithCredentials: Page;
+
         user: CredentialsWithDisplayName;
         displayName?: string;
         app: ElementAppPage;
@@ -163,7 +173,8 @@ export const test = base.extend<
         });
     },
     labsFlags: [],
-    user: async ({ page, homeserver, credentials }, use) => {
+
+    pageWithCredentials: async ({ page, homeserver, credentials }, use) => {
         await page.addInitScript(
             ({ baseUrl, credentials }) => {
                 // Seed the localStorage with the required credentials
@@ -180,9 +191,12 @@ export const test = base.extend<
             },
             { baseUrl: homeserver.config.baseUrl, credentials },
         );
+        await use(page);
+    },
+
+    user: async ({ pageWithCredentials: page, credentials }, use) => {
         await page.goto("/");
         await page.waitForSelector(".mx_MatrixChat", { timeout: 30000 });
-
         await use(credentials);
     },
 

res/css/structures/auth/_LoginSplashView.pcss

@@ -14,6 +14,23 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+.mx_LoginSplashView_migrationProgress {
+    display: flex;
+    flex-direction: column;
+
+    align-items: center;
+    justify-content: center;
+    width: 100%;
+    height: 100%;
+
+    .mx_ProgressBar {
+        height: 8px;
+        width: 600px;
+
+        @mixin ProgressBarBorderRadius 8px;
+    }
+}
+
 .mx_LoginSplashView_splashButtons {
     text-align: center;
     width: 100%;

src/components/structures/MatrixChat.tsx

@@ -2123,6 +2123,7 @@ export default class MatrixChat extends React.PureComponent<IProps, IState> {
                 // Suppress `InvalidStoreError`s here, since they have their own error dialog.
                 view = (
                     <LoginSplashView
+                        matrixClient={MatrixClientPeg.safeGet()}
                         onLogoutClick={this.onLogoutClick}
                         syncError={isStoreError ? null : this.state.syncError}
                     />

src/components/structures/auth/LoginSplashView.tsx

@@ -15,13 +15,19 @@ limitations under the License.
 */
 
 import React from "react";
+import { CryptoEvent, MatrixClient } from "matrix-js-sdk/src/matrix";
 
 import { messageForSyncError } from "../../../utils/ErrorUtils";
 import Spinner from "../../views/elements/Spinner";
+import ProgressBar from "../../views/elements/ProgressBar";
 import AccessibleButton, { ButtonEvent } from "../../views/elements/AccessibleButton";
 import { _t } from "../../../languageHandler";
+import { useTypedEventEmitterState } from "../../../hooks/useEventEmitter";
 
 interface Props {
+    /** The matrix client which is logging in */
+    matrixClient: MatrixClient;
+
     /**
      * A callback function. Will be called if the user clicks the "logout" button on the splash screen.
      *
@@ -35,19 +41,42 @@ interface Props {
     syncError: Error | null;
 }
 
+type MigrationState = {
+    progress: number;
+    totalSteps: number;
+};
+
 /**
  * The view that is displayed after we have logged in, before the first /sync is completed.
  */
 export function LoginSplashView(props: Props): React.JSX.Element {
+    const migrationState = useTypedEventEmitterState(
+        props.matrixClient,
+        CryptoEvent.LegacyCryptoStoreMigrationProgress,
+        (progress?: number, total?: number): MigrationState => ({ progress: progress ?? -1, totalSteps: total ?? -1 }),
+    );
     let errorBox: React.JSX.Element | undefined;
-
     if (props.syncError) {
         errorBox = <div className="mx_LoginSplashView_syncError">{messageForSyncError(props.syncError)}</div>;
     }
+
+    // If we are migrating the crypto data, show a progress bar. Otherwise, show a normal spinner.
+    let spinnerOrProgress;
+    if (migrationState.totalSteps !== -1) {
+        spinnerOrProgress = (
+            <div className="mx_LoginSplashView_migrationProgress">
+                <p>{_t("migrating_crypto")}</p>
+                <ProgressBar value={migrationState.progress} max={migrationState.totalSteps} />
+            </div>
+        );
+    } else {
+        spinnerOrProgress = <Spinner />;
+    }
+
     return (
         <div className="mx_MatrixChat_splash">
             {errorBox}
-            <Spinner />
+            {spinnerOrProgress}
             <div className="mx_LoginSplashView_splashButtons">
                 <AccessibleButton kind="link_inline" onClick={props.onLogoutClick}>
                     {_t("action|logout")}

src/i18n/strings/en_EN.json

@@ -1587,6 +1587,7 @@
     },
     "member_list_back_action_label": "Room members",
     "message_edit_dialog_title": "Message edits",
+    "migrating_crypto": "Hang tight. We are updating Element to make encryption faster and more reliable.",
     "mobile_guide": {
         "toast_accept": "Use app",
         "toast_description": "%(brand)s is experimental on a mobile web browser. For a better experience and the latest features, use our free native app.",

test/components/structures/auth/LoginSplashView-test.tsx

@@ -14,14 +14,24 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { render, RenderResult } from "@testing-library/react";
+import { act, render, RenderResult } from "@testing-library/react";
 import React, { ComponentProps } from "react";
+import EventEmitter from "events";
+import { CryptoEvent } from "matrix-js-sdk/src/matrix";
+import { sleep } from "matrix-js-sdk/src/utils";
 
 import { LoginSplashView } from "../../../../src/components/structures/auth/LoginSplashView";
+import type { MatrixClient } from "matrix-js-sdk/src/matrix";
 
 describe("<LoginSplashView />", () => {
+    let matrixClient: MatrixClient;
+    beforeEach(() => {
+        matrixClient = new EventEmitter() as unknown as MatrixClient;
+    });
+
     function getComponent(props: Partial<ComponentProps<typeof LoginSplashView>> = {}): RenderResult {
         const defaultProps = {
+            matrixClient,
             onLogoutClick: () => {},
             syncError: null,
         };
@@ -46,4 +56,20 @@ describe("<LoginSplashView />", () => {
         rendered.getByRole("button", { name: "Logout" }).click();
         expect(onLogoutClick).toHaveBeenCalled();
     });
+
+    it("Shows migration progress", async () => {
+        const rendered = getComponent();
+
+        act(() => {
+            matrixClient.emit(CryptoEvent.LegacyCryptoStoreMigrationProgress, 5, 10);
+        });
+        rendered.getByText("Hang tight.", { exact: false });
+
+        // Wait for the animation to update
+        await act(() => sleep(500));
+
+        const progress = rendered.getByRole("progressbar");
+        expect(progress.getAttribute("value")).toEqual("5");
+        expect(progress.getAttribute("max")).toEqual("10");
+    });
 });