Wire up module loading to application startup (#21703)

* Early module loader bundler

* Add a module installer script

* Add dev-friendly docs

* Add real module-api dependency

* Speed up `yarn add` for mulitple modules

* Fix version check for modules

* Appease the linter

.eslintignore

@@ -1,3 +1,5 @@
 src/vector/modernizr.js
 # Legacy skinning file that some people might still have
 src/component-index.js
+# Auto-generated file
+src/modules.ts

.eslintrc.js

@@ -18,7 +18,7 @@ module.exports = {
         }
     },
     overrides: [{
-        files: ["src/**/*.{ts,tsx}"],
+        files: ["src/**/*.{ts,tsx}", "module_system/**/*.{ts,tsx}"],
         extends: [
             "plugin:matrix-org/typescript",
             "plugin:matrix-org/react",

.gitignore

@@ -24,3 +24,6 @@ electron/pub
 .vscode/
 .env
 /coverage
+# Auto-generated file
+/src/modules.ts
+/build_config.yaml

build_config.sample.yaml

@@ -0,0 +1,25 @@
+# A sample build_config.yaml to supply to Element Web's build pipeline,
+# enabling custom functionality at compile time. Copy this file to
+# `build_config.yaml` in the same directory to use, as you would with
+# `config.json`.
+#
+# Note: The vast majority of users DO NOT need this. If you are looking
+# to build your own Element Web as seen on app.element.io or similar then
+# this is not required.
+#
+# This config file does become required if you are looking to add runtime
+# functionality to Element Web, such as customisation endpoints and modules.
+#
+# Over time we might expand this config to better support some use cases.
+# Watch the release notes for features which might impact this config.
+
+# The modules to install. See ./docs/modules.md for more information on
+# what modules are.
+#
+# The values of this are provided to `yarn add` for inclusion.
+modules:
+    # An example of pulling a module from NPM
+    - "@vector-im/element-web-ilag-module@^0.0.1"
+
+    # An example of pulling a module from github
+    - "github:vector-im/element-web-ilag-module#main"

docs/modules.md

@@ -0,0 +1,48 @@
+# Module system
+
+The module system in Element Web is a way to add or modify functionality of Element Web itself, bundled at compile time
+for the app. This means that modules are loaded as part of the `yarn build` process but have an effect on user experience
+at runtime.
+
+## Installing modules
+
+If you already have a module you want to install, such as our [ILAG Module](https://github.com/vector-im/element-web-ilag-module),
+then copy `build_config.sample.yaml` to `build_config.yaml` in the same directory. In your new `build_config.yaml` simply
+add the reference to the module as described by the sample file, using the same syntax you would for `yarn add`:
+
+```yaml
+modules:
+    # Our module happens to be published on NPM, so we use that syntax to reference it.
+    - "@vector-im/element-web-ilag-module@latest"
+```
+
+Then build the app as you normally would: `yarn build` or `yarn dist` (if compatible on your platform). If you are building
+the Docker image then ensure your `build_config.yaml` ends up in the build directory. Usually this works fine if you use
+the current directory as the build context (the `.` in `docker build -t my-element-web .`).
+
+## Writing modules
+
+While writing modules is meant to be easy, not everything is possible yet. For modules which want to do something we haven't
+exposed in the module API, the module API will need to be updated. This means a PR to both the
+[`matrix-react-sdk`](https://github.com/matrix-org/matrix-react-sdk) and [`matrix-react-sdk-module-api`](https://github.com/matrix-org/matrix-react-sdk-module-api).
+
+Once your change to the module API is accepted, the `@matrix-org/react-sdk-module-api` dependency gets updated at the
+`matrix-react-sdk` and `element-web` layers (usually by us, the maintainers) to ensure your module can operate.
+
+If you're not adding anything to the module API, or your change was accepted per above, then start off with a clone of
+our [ILAG module](https://github.com/vector-im/element-web-ilag-module) which will give you a general idea for what the
+structure of a module is and how it works.
+
+The following requirements are key for any module:
+1. The module must depend on `@matrix-org/react-sdk-module-api` (usually as a dev dependency).
+2. The module's `main` entrypoint must have a `default` export for the `RuntimeModule` instance, supporting a constructor
+   which takes a single parameter: a `ModuleApi` instance. This instance is passed to `super()`.
+3. The module must be deployed in a way where `yarn add` can access it, as that is how the build system will try to
+   install it. Note that while this is often NPM, it can also be a GitHub/GitLab repo or private NPM registry.
+
+... and that's pretty much it. As with any code, please be responsible and call things in line with the documentation.
+Both `RuntimeModule` and `ModuleApi` have extensive documentation to describe what is proper usage and how to set things
+up.
+
+If you have any questions then please visit [#element-dev:matrix.org](https://matrix.to/#/#element-dev:matrix.org) on
+Matrix and we'll help as best we can.

module_system/BuildConfig.ts

@@ -0,0 +1,33 @@
+/*
+Copyright 2022 New Vector Ltd.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import * as YAML from "yaml";
+import * as fs from "fs";
+
+export type BuildConfig = {
+    // Dev note: make everything here optional for user safety. Invalid
+    // configs are very possible.
+
+    // The module references to include in the build.
+    modules?: string[];
+};
+
+export function readBuildConfig(): BuildConfig {
+    if (fs.existsSync("./build_config.yaml")) {
+        return YAML.parse(fs.readFileSync("./build_config.yaml", "utf-8"));
+    }
+    return {}; // no config
+}

module_system/installer.ts

@@ -0,0 +1,191 @@
+/*
+Copyright 2022 New Vector Ltd.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import * as fs from "fs";
+import * as childProcess from "child_process";
+import * as semver from "semver";
+
+import { BuildConfig } from "./BuildConfig";
+
+// This expects to be run from ./scripts/install.ts
+
+const moduleApiDepName = "@matrix-org/react-sdk-module-api";
+
+const MODULES_TS_HEADER = `
+/*
+ * THIS FILE IS AUTO-GENERATED
+ * You can edit it you like, but your changes will be overwritten,
+ * so you'd just be trying to swim upstream like a salmon.
+ * You are not a salmon.
+ */
+
+import { RuntimeModule } from "@matrix-org/react-sdk-module-api/lib/RuntimeModule";
+`;
+const MODULES_TS_DEFINITIONS = `
+export const INSTALLED_MODULES: RuntimeModule[] = [];
+`;
+
+export function installer(config: BuildConfig): void {
+    if (!config.modules?.length) {
+        // nothing to do
+        writeModulesTs(MODULES_TS_HEADER + MODULES_TS_DEFINITIONS);
+        return;
+    }
+
+    let exitCode = 0;
+
+    // We cheat a bit and store the current package.json and lockfile so we can safely
+    // run `yarn add` without creating extra committed files for people. We restore
+    // these files by simply overwriting them when we're done.
+    const packageDeps = readCurrentPackageDetails();
+
+    // Record which optional dependencies there are currently, if any, so we can exclude
+    // them from our "must be a module" assumption later on.
+    const currentOptDeps = getOptionalDepNames(packageDeps.packageJson);
+
+    try {
+        // Install the modules with yarn
+        const yarnAddRef = config.modules.join(" ");
+        callYarnAdd(yarnAddRef); // install them all at once
+
+        // Grab the optional dependencies again and exclude what was there already. Everything
+        // else must be a module, we assume.
+        const pkgJsonStr = fs.readFileSync("./package.json", "utf-8");
+        const optionalDepNames = getOptionalDepNames(pkgJsonStr);
+        const installedModules = optionalDepNames.filter(d => !currentOptDeps.includes(d));
+
+        // Ensure all the modules are compatible. We check them all and report at the end to
+        // try and save the user some time debugging this sort of failure.
+        const ourApiVersion = getTopLevelDependencyVersion(moduleApiDepName);
+        const incompatibleNames: string[] = [];
+        for (const moduleName of installedModules) {
+            const modApiVersion = getModuleApiVersionFor(moduleName);
+            if (!isModuleVersionCompatible(ourApiVersion, modApiVersion)) {
+                incompatibleNames.push(moduleName);
+            }
+        }
+        if (incompatibleNames.length > 0) {
+            console.error(
+                "The following modules are not compatible with this version of element-web. Please update the module " +
+                "references and try again.",
+                JSON.stringify(incompatibleNames, null, 4), // stringify to get prettier/complete output
+            );
+            exitCode = 1;
+            return; // hit the finally{} block before exiting
+        }
+
+        // If we reach here, everything seems fine. Write modules.ts and log some output
+        // Note: we compile modules.ts in two parts for developer friendliness if they
+        // happen to look at it.
+        console.log("The following modules have been installed: ", installedModules);
+        let modulesTsHeader = MODULES_TS_HEADER;
+        let modulesTsDefs = MODULES_TS_DEFINITIONS;
+        let index = 0;
+        for (const moduleName of installedModules) {
+            const importName = `Module${++index}`;
+            modulesTsHeader += `import ${importName} from "${moduleName}";\n`;
+            modulesTsDefs += `INSTALLED_MODULES.push(${importName});\n`;
+        }
+        writeModulesTs(modulesTsHeader + modulesTsDefs);
+        console.log("Done installing modules");
+    } finally {
+        // Always restore package details (or at least try to)
+        writePackageDetails(packageDeps);
+
+        if (exitCode > 0) {
+            process.exit(exitCode);
+        }
+    }
+}
+
+type RawDependencies = {
+    lockfile: string;
+    packageJson: string;
+};
+
+function readCurrentPackageDetails(): RawDependencies {
+    return {
+        lockfile: fs.readFileSync("./yarn.lock", "utf-8"),
+        packageJson: fs.readFileSync("./package.json", "utf-8"),
+    };
+}
+
+function writePackageDetails(deps: RawDependencies) {
+    fs.writeFileSync("./yarn.lock", deps.lockfile, "utf-8");
+    fs.writeFileSync("./package.json", deps.packageJson, "utf-8");
+}
+
+function callYarnAdd(dep: string) {
+    // Add the module to the optional dependencies section just in case something
+    // goes wrong in restoring the original package details.
+    childProcess.execSync(`yarn add -O ${dep}`, {
+        env: process.env,
+        stdio: ['inherit', 'inherit', 'inherit'],
+    });
+}
+
+function getOptionalDepNames(pkgJsonStr: string): string[] {
+    return Object.keys(JSON.parse(pkgJsonStr)?.['optionalDependencies'] ?? {});
+}
+
+function findDepVersionInPackageJson(dep: string, pkgJsonStr: string): string {
+    const pkgJson = JSON.parse(pkgJsonStr);
+    const packages = {
+        ...(pkgJson['optionalDependencies'] ?? {}),
+        ...(pkgJson['devDependencies'] ?? {}),
+        ...(pkgJson['dependencies'] ?? {}),
+    };
+    return packages[dep];
+}
+
+function getTopLevelDependencyVersion(dep: string): string {
+    const dependencyTree = JSON.parse(childProcess.execSync(`npm list ${dep} --depth=0 --json`, {
+        env: process.env,
+        stdio: ['inherit', 'pipe', 'pipe'],
+    }).toString('utf-8'));
+
+    /*
+        What a dependency tree looks like:
+        {
+          "version": "1.10.13",
+          "name": "element-web",
+          "dependencies": {
+            "@matrix-org/react-sdk-module-api": {
+              "version": "0.0.1",
+              "resolved": "file:../../../matrix-react-sdk-module-api"
+            }
+          }
+        }
+     */
+
+    return dependencyTree["dependencies"][dep]["version"];
+}
+
+function getModuleApiVersionFor(moduleName: string): string {
+    // We'll just pretend that this isn't highly problematic...
+    // Yarn is fairly stable in putting modules in a flat hierarchy, at least.
+    const pkgJsonStr = fs.readFileSync(`./node_modules/${moduleName}/package.json`, "utf-8");
+    return findDepVersionInPackageJson(moduleApiDepName, pkgJsonStr);
+}
+
+function isModuleVersionCompatible(ourApiVersion: string, moduleApiVersion: string): boolean {
+    if (!moduleApiVersion) return false;
+    return semver.satisfies(ourApiVersion, moduleApiVersion);
+}
+
+function writeModulesTs(content: string) {
+    fs.writeFileSync("./src/modules.ts", content, "utf-8");
+}

module_system/scripts/install.ts

@@ -0,0 +1,21 @@
+/*
+Copyright 2022 New Vector Ltd.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { readBuildConfig } from "../BuildConfig";
+import { installer } from "../installer";
+
+const buildConf = readBuildConfig();
+installer(buildConf);

package.json

@@ -35,25 +35,27 @@
     "build-stats": "yarn clean && yarn build:genfiles && yarn build:bundle-stats",
     "build:jitsi": "node scripts/build-jitsi.js",
     "build:res": "node scripts/copy-res.js",
-    "build:genfiles": "yarn build:res && yarn build:jitsi",
+    "build:genfiles": "yarn build:res && yarn build:jitsi && yarn build:module_system",
     "build:modernizr": "modernizr -c .modernizr.json -d src/vector/modernizr.js",
     "build:bundle": "webpack --progress --bail --mode production",
     "build:bundle-stats": "webpack --progress --bail --mode production --json > webpack-stats.json",
+    "build:module_system": "tsc --project ./tsconfig.module_system.json && node ./lib/module_system/scripts/install.js",
     "dist": "scripts/package.sh",
     "start": "concurrently --kill-others-on-fail --prefix \"{time} [{name}]\" -n res,element-js \"yarn start:res\" \"yarn start:js\"",
     "start:https": "concurrently --kill-others-on-fail --prefix \"{time} [{name}]\" -n res,element-js \"yarn start:res\" \"yarn start:js --https\"",
     "start:res": "yarn build:jitsi && node scripts/copy-res.js -w",
     "start:js": "webpack-dev-server --host=0.0.0.0 --output-filename=bundles/_dev_/[name].js --output-chunk-filename=bundles/_dev_/[name].js -w --mode development --disable-host-check --hot",
     "lint": "yarn lint:types && yarn lint:js && yarn lint:style",
-    "lint:js": "eslint --max-warnings 0 src",
+    "lint:js": "eslint --max-warnings 0 src module_system",
-    "lint:js-fix": "eslint --fix src",
+    "lint:js-fix": "eslint --fix src module_system",
-    "lint:types": "tsc --noEmit --jsx react",
+    "lint:types": "tsc --noEmit --jsx react && tsc --noEmit --project ./tsconfig.module_system.json",
     "lint:style": "stylelint \"res/css/**/*.scss\"",
     "test": "jest",
     "coverage": "yarn test --coverage"
   },
   "dependencies": {
     "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz",
+    "@matrix-org/react-sdk-module-api": "^0.0.3",
     "browser-request": "^0.3.3",
     "gfm.css": "^1.1.2",
     "jsrsasign": "^10.5.25",
@@ -144,6 +146,7 @@
     "postcss-strip-inline-comments": "^0.1.5",
     "raw-loader": "^4.0.2",
     "rimraf": "^3.0.2",
+    "semver": "^7.3.7",
     "shell-escape": "^0.2.0",
     "simple-proxy-agent": "^1.1.0",
     "string-replace-loader": "2",
@@ -157,7 +160,8 @@
     "webpack-cli": "^3.3.12",
     "webpack-dev-server": "^3.11.2",
     "worker-loader": "^2.0.0",
-    "worklet-loader": "^2.0.0"
+    "worklet-loader": "^2.0.0",
+    "yaml": "^2.0.1"
   },
   "resolutions": {
     "@types/react": "17.0.14"

src/vector/index.ts

@@ -113,6 +113,7 @@ async function start() {
         loadLanguage,
         loadTheme,
         loadApp,
+        loadModules,
         showError,
         showIncompatibleBrowser,
         _t,
@@ -155,6 +156,11 @@ async function start() {
         // now that the config is ready, try to persist logs
         const persistLogsPromise = setupLogStorage();
 
+        // Load modules before language to ensure any custom translations are respected, and any app
+        // startup functionality is run
+        const loadModulesPromise = loadModules();
+        await settled(loadModulesPromise);
+
         // Load language after loading config.json so that settingsDefaults.language can be applied
         const loadLanguagePromise = loadLanguage();
         // as quickly as we possibly can, set a default theme...
@@ -209,6 +215,7 @@ async function start() {
         // assert things started successfully
         // ##################################
         await loadOlmPromise;
+        await loadModulesPromise;
         await loadThemePromise;
         await loadLanguagePromise;
 

src/vector/init.tsx

@@ -2,7 +2,7 @@
 Copyright 2015, 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
 Copyright 2019 Michael Telatynski <7t3chguy@gmail.com>
-Copyright 2018 - 2021 New Vector Ltd
+Copyright 2018 - 2022 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -29,11 +29,15 @@ import PlatformPeg from "matrix-react-sdk/src/PlatformPeg";
 import SdkConfig from "matrix-react-sdk/src/SdkConfig";
 import { setTheme } from "matrix-react-sdk/src/theme";
 import { logger } from "matrix-js-sdk/src/logger";
+import { ModuleRunner } from "matrix-react-sdk/src/modules/ModuleRunner";
 
 import ElectronPlatform from "./platform/ElectronPlatform";
 import PWAPlatform from "./platform/PWAPlatform";
 import WebPlatform from "./platform/WebPlatform";
 import { initRageshake, initRageshakeStore } from "./rageshakesetup";
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore - this path is created at runtime and therefore won't exist at typecheck time
+import { INSTALLED_MODULES } from "../modules";
 
 export const rageshakePromise = initRageshake();
 
@@ -157,4 +161,12 @@ export async function showIncompatibleBrowser(onAccept) {
         document.getElementById('matrixchat'));
 }
 
+export async function loadModules() {
+    for (const InstalledModule of INSTALLED_MODULES) {
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-ignore - we know the constructor exists even if TypeScript can't be convinced of that
+        ModuleRunner.instance.registerModule((api) => new InstalledModule(api));
+    }
+}
+
 export const _t = languageHandler._t;

tsconfig.module_system.json

@@ -0,0 +1,14 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "jsx": "preserve",
+    "declaration": false,
+    "outDir": "./lib/module_system",
+    "lib": [
+      "es2019"
+    ]
+  },
+  "include": [
+    "./module_system/**/*.ts"
+  ]
+}

yarn.lock

@@ -1094,6 +1094,13 @@
   dependencies:
     regenerator-runtime "^0.13.4"
 
+"@babel/runtime@^7.17.9":
+  version "7.18.3"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.18.3.tgz#c7b654b57f6f63cf7f8b418ac9ca04408c4579f4"
+  integrity sha512-38Y8f7YUhce/K7RMwTp7m0uCumpv9hZkitCbBClqQIow1qSbCvGkcegKOXpEWCQLfWmevgRiWokZ1GkpfhbZug==
+  dependencies:
+    regenerator-runtime "^0.13.4"
+
 "@babel/template@^7.16.7", "@babel/template@^7.3.3":
   version "7.16.7"
   resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.16.7.tgz#8d126c8701fde4d66b264b3eba3d96f07666d155"
@@ -1514,6 +1521,13 @@
   version "3.2.8"
   resolved "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz#8d53636d045e1776e2a2ec6613e57330dd9ce856"
 
+"@matrix-org/react-sdk-module-api@^0.0.3":
+  version "0.0.3"
+  resolved "https://registry.yarnpkg.com/@matrix-org/react-sdk-module-api/-/react-sdk-module-api-0.0.3.tgz#a7ac1b18a72d18d08290b81fa33b0d8d00a77d2b"
+  integrity sha512-jQmLhVIanuX0g7Jx1OIqlzs0kp72PfSpv3umi55qVPYcAPQmO252AUs0vncatK8O4e013vohdnNhly19a/kmLQ==
+  dependencies:
+    "@babel/runtime" "^7.17.9"
+
 "@mrmlnc/readdir-enhanced@^2.2.1":
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz#524af240d1a360527b730475ecfa1344aa540dde"
@@ -13436,6 +13450,11 @@ yaml@^1.10.0:
   resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b"
   integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==
 
+yaml@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.0.1.tgz#71886d6021f3da28169dbefde78d4dd0f8d83650"
+  integrity sha512-1NpAYQ3wjzIlMs0mgdBmYzLkFgWBIWrzYVDYfrixhoFNNgJ444/jT2kUT2sicRbJES3oQYRZugjB6Ro8SjKeFg==
+
 yargs-parser@^13.1.2:
   version "13.1.2"
   resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-13.1.2.tgz#130f09702ebaeef2650d54ce6e3e5706f7a4fb38"