From f1e5b95554b2c7f1286d6b50a11581b0da65d950 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 5 Jul 2022 20:26:54 +0200 Subject: [PATCH] Wire up module loading to application startup (#21703) * Early module loader bundler * Add a module installer script * Add dev-friendly docs * Add real module-api dependency * Speed up `yarn add` for mulitple modules * Fix version check for modules * Appease the linter --- .eslintignore | 2 + .eslintrc.js | 2 +- .gitignore | 3 + build_config.sample.yaml | 25 ++++ docs/modules.md | 48 ++++++++ module_system/BuildConfig.ts | 33 ++++++ module_system/installer.ts | 191 +++++++++++++++++++++++++++++++ module_system/scripts/install.ts | 21 ++++ package.json | 14 ++- src/vector/index.ts | 7 ++ src/vector/init.tsx | 14 ++- tsconfig.module_system.json | 14 +++ yarn.lock | 19 +++ 13 files changed, 386 insertions(+), 7 deletions(-) create mode 100644 build_config.sample.yaml create mode 100644 docs/modules.md create mode 100644 module_system/BuildConfig.ts create mode 100644 module_system/installer.ts create mode 100644 module_system/scripts/install.ts create mode 100644 tsconfig.module_system.json diff --git a/.eslintignore b/.eslintignore index e5cc476988..8a5f170a5f 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,3 +1,5 @@ src/vector/modernizr.js # Legacy skinning file that some people might still have src/component-index.js +# Auto-generated file +src/modules.ts diff --git a/.eslintrc.js b/.eslintrc.js index b6f2693ab5..a3dd5073cf 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -18,7 +18,7 @@ module.exports = { } }, overrides: [{ - files: ["src/**/*.{ts,tsx}"], + files: ["src/**/*.{ts,tsx}", "module_system/**/*.{ts,tsx}"], extends: [ "plugin:matrix-org/typescript", "plugin:matrix-org/react", diff --git a/.gitignore b/.gitignore index 9d86780deb..b279ef0bf9 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ electron/pub .vscode/ .env /coverage +# Auto-generated file +/src/modules.ts +/build_config.yaml diff --git a/build_config.sample.yaml b/build_config.sample.yaml new file mode 100644 index 0000000000..a41e991a59 --- /dev/null +++ b/build_config.sample.yaml @@ -0,0 +1,25 @@ +# A sample build_config.yaml to supply to Element Web's build pipeline, +# enabling custom functionality at compile time. Copy this file to +# `build_config.yaml` in the same directory to use, as you would with +# `config.json`. +# +# Note: The vast majority of users DO NOT need this. If you are looking +# to build your own Element Web as seen on app.element.io or similar then +# this is not required. +# +# This config file does become required if you are looking to add runtime +# functionality to Element Web, such as customisation endpoints and modules. +# +# Over time we might expand this config to better support some use cases. +# Watch the release notes for features which might impact this config. + +# The modules to install. See ./docs/modules.md for more information on +# what modules are. +# +# The values of this are provided to `yarn add` for inclusion. +modules: + # An example of pulling a module from NPM + - "@vector-im/element-web-ilag-module@^0.0.1" + + # An example of pulling a module from github + - "github:vector-im/element-web-ilag-module#main" diff --git a/docs/modules.md b/docs/modules.md new file mode 100644 index 0000000000..d5c101d16f --- /dev/null +++ b/docs/modules.md @@ -0,0 +1,48 @@ +# Module system + +The module system in Element Web is a way to add or modify functionality of Element Web itself, bundled at compile time +for the app. This means that modules are loaded as part of the `yarn build` process but have an effect on user experience +at runtime. + +## Installing modules + +If you already have a module you want to install, such as our [ILAG Module](https://github.com/vector-im/element-web-ilag-module), +then copy `build_config.sample.yaml` to `build_config.yaml` in the same directory. In your new `build_config.yaml` simply +add the reference to the module as described by the sample file, using the same syntax you would for `yarn add`: + +```yaml +modules: + # Our module happens to be published on NPM, so we use that syntax to reference it. + - "@vector-im/element-web-ilag-module@latest" +``` + +Then build the app as you normally would: `yarn build` or `yarn dist` (if compatible on your platform). If you are building +the Docker image then ensure your `build_config.yaml` ends up in the build directory. Usually this works fine if you use +the current directory as the build context (the `.` in `docker build -t my-element-web .`). + +## Writing modules + +While writing modules is meant to be easy, not everything is possible yet. For modules which want to do something we haven't +exposed in the module API, the module API will need to be updated. This means a PR to both the +[`matrix-react-sdk`](https://github.com/matrix-org/matrix-react-sdk) and [`matrix-react-sdk-module-api`](https://github.com/matrix-org/matrix-react-sdk-module-api). + +Once your change to the module API is accepted, the `@matrix-org/react-sdk-module-api` dependency gets updated at the +`matrix-react-sdk` and `element-web` layers (usually by us, the maintainers) to ensure your module can operate. + +If you're not adding anything to the module API, or your change was accepted per above, then start off with a clone of +our [ILAG module](https://github.com/vector-im/element-web-ilag-module) which will give you a general idea for what the +structure of a module is and how it works. + +The following requirements are key for any module: +1. The module must depend on `@matrix-org/react-sdk-module-api` (usually as a dev dependency). +2. The module's `main` entrypoint must have a `default` export for the `RuntimeModule` instance, supporting a constructor + which takes a single parameter: a `ModuleApi` instance. This instance is passed to `super()`. +3. The module must be deployed in a way where `yarn add` can access it, as that is how the build system will try to + install it. Note that while this is often NPM, it can also be a GitHub/GitLab repo or private NPM registry. + +... and that's pretty much it. As with any code, please be responsible and call things in line with the documentation. +Both `RuntimeModule` and `ModuleApi` have extensive documentation to describe what is proper usage and how to set things +up. + +If you have any questions then please visit [#element-dev:matrix.org](https://matrix.to/#/#element-dev:matrix.org) on +Matrix and we'll help as best we can. diff --git a/module_system/BuildConfig.ts b/module_system/BuildConfig.ts new file mode 100644 index 0000000000..59e17d0da9 --- /dev/null +++ b/module_system/BuildConfig.ts @@ -0,0 +1,33 @@ +/* +Copyright 2022 New Vector Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import * as YAML from "yaml"; +import * as fs from "fs"; + +export type BuildConfig = { + // Dev note: make everything here optional for user safety. Invalid + // configs are very possible. + + // The module references to include in the build. + modules?: string[]; +}; + +export function readBuildConfig(): BuildConfig { + if (fs.existsSync("./build_config.yaml")) { + return YAML.parse(fs.readFileSync("./build_config.yaml", "utf-8")); + } + return {}; // no config +} diff --git a/module_system/installer.ts b/module_system/installer.ts new file mode 100644 index 0000000000..021550fdee --- /dev/null +++ b/module_system/installer.ts @@ -0,0 +1,191 @@ +/* +Copyright 2022 New Vector Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import * as fs from "fs"; +import * as childProcess from "child_process"; +import * as semver from "semver"; + +import { BuildConfig } from "./BuildConfig"; + +// This expects to be run from ./scripts/install.ts + +const moduleApiDepName = "@matrix-org/react-sdk-module-api"; + +const MODULES_TS_HEADER = ` +/* + * THIS FILE IS AUTO-GENERATED + * You can edit it you like, but your changes will be overwritten, + * so you'd just be trying to swim upstream like a salmon. + * You are not a salmon. + */ + +import { RuntimeModule } from "@matrix-org/react-sdk-module-api/lib/RuntimeModule"; +`; +const MODULES_TS_DEFINITIONS = ` +export const INSTALLED_MODULES: RuntimeModule[] = []; +`; + +export function installer(config: BuildConfig): void { + if (!config.modules?.length) { + // nothing to do + writeModulesTs(MODULES_TS_HEADER + MODULES_TS_DEFINITIONS); + return; + } + + let exitCode = 0; + + // We cheat a bit and store the current package.json and lockfile so we can safely + // run `yarn add` without creating extra committed files for people. We restore + // these files by simply overwriting them when we're done. + const packageDeps = readCurrentPackageDetails(); + + // Record which optional dependencies there are currently, if any, so we can exclude + // them from our "must be a module" assumption later on. + const currentOptDeps = getOptionalDepNames(packageDeps.packageJson); + + try { + // Install the modules with yarn + const yarnAddRef = config.modules.join(" "); + callYarnAdd(yarnAddRef); // install them all at once + + // Grab the optional dependencies again and exclude what was there already. Everything + // else must be a module, we assume. + const pkgJsonStr = fs.readFileSync("./package.json", "utf-8"); + const optionalDepNames = getOptionalDepNames(pkgJsonStr); + const installedModules = optionalDepNames.filter(d => !currentOptDeps.includes(d)); + + // Ensure all the modules are compatible. We check them all and report at the end to + // try and save the user some time debugging this sort of failure. + const ourApiVersion = getTopLevelDependencyVersion(moduleApiDepName); + const incompatibleNames: string[] = []; + for (const moduleName of installedModules) { + const modApiVersion = getModuleApiVersionFor(moduleName); + if (!isModuleVersionCompatible(ourApiVersion, modApiVersion)) { + incompatibleNames.push(moduleName); + } + } + if (incompatibleNames.length > 0) { + console.error( + "The following modules are not compatible with this version of element-web. Please update the module " + + "references and try again.", + JSON.stringify(incompatibleNames, null, 4), // stringify to get prettier/complete output + ); + exitCode = 1; + return; // hit the finally{} block before exiting + } + + // If we reach here, everything seems fine. Write modules.ts and log some output + // Note: we compile modules.ts in two parts for developer friendliness if they + // happen to look at it. + console.log("The following modules have been installed: ", installedModules); + let modulesTsHeader = MODULES_TS_HEADER; + let modulesTsDefs = MODULES_TS_DEFINITIONS; + let index = 0; + for (const moduleName of installedModules) { + const importName = `Module${++index}`; + modulesTsHeader += `import ${importName} from "${moduleName}";\n`; + modulesTsDefs += `INSTALLED_MODULES.push(${importName});\n`; + } + writeModulesTs(modulesTsHeader + modulesTsDefs); + console.log("Done installing modules"); + } finally { + // Always restore package details (or at least try to) + writePackageDetails(packageDeps); + + if (exitCode > 0) { + process.exit(exitCode); + } + } +} + +type RawDependencies = { + lockfile: string; + packageJson: string; +}; + +function readCurrentPackageDetails(): RawDependencies { + return { + lockfile: fs.readFileSync("./yarn.lock", "utf-8"), + packageJson: fs.readFileSync("./package.json", "utf-8"), + }; +} + +function writePackageDetails(deps: RawDependencies) { + fs.writeFileSync("./yarn.lock", deps.lockfile, "utf-8"); + fs.writeFileSync("./package.json", deps.packageJson, "utf-8"); +} + +function callYarnAdd(dep: string) { + // Add the module to the optional dependencies section just in case something + // goes wrong in restoring the original package details. + childProcess.execSync(`yarn add -O ${dep}`, { + env: process.env, + stdio: ['inherit', 'inherit', 'inherit'], + }); +} + +function getOptionalDepNames(pkgJsonStr: string): string[] { + return Object.keys(JSON.parse(pkgJsonStr)?.['optionalDependencies'] ?? {}); +} + +function findDepVersionInPackageJson(dep: string, pkgJsonStr: string): string { + const pkgJson = JSON.parse(pkgJsonStr); + const packages = { + ...(pkgJson['optionalDependencies'] ?? {}), + ...(pkgJson['devDependencies'] ?? {}), + ...(pkgJson['dependencies'] ?? {}), + }; + return packages[dep]; +} + +function getTopLevelDependencyVersion(dep: string): string { + const dependencyTree = JSON.parse(childProcess.execSync(`npm list ${dep} --depth=0 --json`, { + env: process.env, + stdio: ['inherit', 'pipe', 'pipe'], + }).toString('utf-8')); + + /* + What a dependency tree looks like: + { + "version": "1.10.13", + "name": "element-web", + "dependencies": { + "@matrix-org/react-sdk-module-api": { + "version": "0.0.1", + "resolved": "file:../../../matrix-react-sdk-module-api" + } + } + } + */ + + return dependencyTree["dependencies"][dep]["version"]; +} + +function getModuleApiVersionFor(moduleName: string): string { + // We'll just pretend that this isn't highly problematic... + // Yarn is fairly stable in putting modules in a flat hierarchy, at least. + const pkgJsonStr = fs.readFileSync(`./node_modules/${moduleName}/package.json`, "utf-8"); + return findDepVersionInPackageJson(moduleApiDepName, pkgJsonStr); +} + +function isModuleVersionCompatible(ourApiVersion: string, moduleApiVersion: string): boolean { + if (!moduleApiVersion) return false; + return semver.satisfies(ourApiVersion, moduleApiVersion); +} + +function writeModulesTs(content: string) { + fs.writeFileSync("./src/modules.ts", content, "utf-8"); +} diff --git a/module_system/scripts/install.ts b/module_system/scripts/install.ts new file mode 100644 index 0000000000..a3de3c46da --- /dev/null +++ b/module_system/scripts/install.ts @@ -0,0 +1,21 @@ +/* +Copyright 2022 New Vector Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { readBuildConfig } from "../BuildConfig"; +import { installer } from "../installer"; + +const buildConf = readBuildConfig(); +installer(buildConf); diff --git a/package.json b/package.json index cd3afdbac0..279d27c12f 100644 --- a/package.json +++ b/package.json @@ -35,25 +35,27 @@ "build-stats": "yarn clean && yarn build:genfiles && yarn build:bundle-stats", "build:jitsi": "node scripts/build-jitsi.js", "build:res": "node scripts/copy-res.js", - "build:genfiles": "yarn build:res && yarn build:jitsi", + "build:genfiles": "yarn build:res && yarn build:jitsi && yarn build:module_system", "build:modernizr": "modernizr -c .modernizr.json -d src/vector/modernizr.js", "build:bundle": "webpack --progress --bail --mode production", "build:bundle-stats": "webpack --progress --bail --mode production --json > webpack-stats.json", + "build:module_system": "tsc --project ./tsconfig.module_system.json && node ./lib/module_system/scripts/install.js", "dist": "scripts/package.sh", "start": "concurrently --kill-others-on-fail --prefix \"{time} [{name}]\" -n res,element-js \"yarn start:res\" \"yarn start:js\"", "start:https": "concurrently --kill-others-on-fail --prefix \"{time} [{name}]\" -n res,element-js \"yarn start:res\" \"yarn start:js --https\"", "start:res": "yarn build:jitsi && node scripts/copy-res.js -w", "start:js": "webpack-dev-server --host=0.0.0.0 --output-filename=bundles/_dev_/[name].js --output-chunk-filename=bundles/_dev_/[name].js -w --mode development --disable-host-check --hot", "lint": "yarn lint:types && yarn lint:js && yarn lint:style", - "lint:js": "eslint --max-warnings 0 src", - "lint:js-fix": "eslint --fix src", - "lint:types": "tsc --noEmit --jsx react", + "lint:js": "eslint --max-warnings 0 src module_system", + "lint:js-fix": "eslint --fix src module_system", + "lint:types": "tsc --noEmit --jsx react && tsc --noEmit --project ./tsconfig.module_system.json", "lint:style": "stylelint \"res/css/**/*.scss\"", "test": "jest", "coverage": "yarn test --coverage" }, "dependencies": { "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz", + "@matrix-org/react-sdk-module-api": "^0.0.3", "browser-request": "^0.3.3", "gfm.css": "^1.1.2", "jsrsasign": "^10.5.25", @@ -144,6 +146,7 @@ "postcss-strip-inline-comments": "^0.1.5", "raw-loader": "^4.0.2", "rimraf": "^3.0.2", + "semver": "^7.3.7", "shell-escape": "^0.2.0", "simple-proxy-agent": "^1.1.0", "string-replace-loader": "2", @@ -157,7 +160,8 @@ "webpack-cli": "^3.3.12", "webpack-dev-server": "^3.11.2", "worker-loader": "^2.0.0", - "worklet-loader": "^2.0.0" + "worklet-loader": "^2.0.0", + "yaml": "^2.0.1" }, "resolutions": { "@types/react": "17.0.14" diff --git a/src/vector/index.ts b/src/vector/index.ts index e27129d9aa..5b60f324cd 100644 --- a/src/vector/index.ts +++ b/src/vector/index.ts @@ -113,6 +113,7 @@ async function start() { loadLanguage, loadTheme, loadApp, + loadModules, showError, showIncompatibleBrowser, _t, @@ -155,6 +156,11 @@ async function start() { // now that the config is ready, try to persist logs const persistLogsPromise = setupLogStorage(); + // Load modules before language to ensure any custom translations are respected, and any app + // startup functionality is run + const loadModulesPromise = loadModules(); + await settled(loadModulesPromise); + // Load language after loading config.json so that settingsDefaults.language can be applied const loadLanguagePromise = loadLanguage(); // as quickly as we possibly can, set a default theme... @@ -209,6 +215,7 @@ async function start() { // assert things started successfully // ################################## await loadOlmPromise; + await loadModulesPromise; await loadThemePromise; await loadLanguagePromise; diff --git a/src/vector/init.tsx b/src/vector/init.tsx index 0e43700ae9..72dcd6b073 100644 --- a/src/vector/init.tsx +++ b/src/vector/init.tsx @@ -2,7 +2,7 @@ Copyright 2015, 2016 OpenMarket Ltd Copyright 2017 Vector Creations Ltd Copyright 2019 Michael Telatynski <7t3chguy@gmail.com> -Copyright 2018 - 2021 New Vector Ltd +Copyright 2018 - 2022 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -29,11 +29,15 @@ import PlatformPeg from "matrix-react-sdk/src/PlatformPeg"; import SdkConfig from "matrix-react-sdk/src/SdkConfig"; import { setTheme } from "matrix-react-sdk/src/theme"; import { logger } from "matrix-js-sdk/src/logger"; +import { ModuleRunner } from "matrix-react-sdk/src/modules/ModuleRunner"; import ElectronPlatform from "./platform/ElectronPlatform"; import PWAPlatform from "./platform/PWAPlatform"; import WebPlatform from "./platform/WebPlatform"; import { initRageshake, initRageshakeStore } from "./rageshakesetup"; +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore - this path is created at runtime and therefore won't exist at typecheck time +import { INSTALLED_MODULES } from "../modules"; export const rageshakePromise = initRageshake(); @@ -157,4 +161,12 @@ export async function showIncompatibleBrowser(onAccept) { document.getElementById('matrixchat')); } +export async function loadModules() { + for (const InstalledModule of INSTALLED_MODULES) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore - we know the constructor exists even if TypeScript can't be convinced of that + ModuleRunner.instance.registerModule((api) => new InstalledModule(api)); + } +} + export const _t = languageHandler._t; diff --git a/tsconfig.module_system.json b/tsconfig.module_system.json new file mode 100644 index 0000000000..2b656d4df1 --- /dev/null +++ b/tsconfig.module_system.json @@ -0,0 +1,14 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "jsx": "preserve", + "declaration": false, + "outDir": "./lib/module_system", + "lib": [ + "es2019" + ] + }, + "include": [ + "./module_system/**/*.ts" + ] +} diff --git a/yarn.lock b/yarn.lock index ed8d41c055..f586f3ca5f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1094,6 +1094,13 @@ dependencies: regenerator-runtime "^0.13.4" +"@babel/runtime@^7.17.9": + version "7.18.3" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.18.3.tgz#c7b654b57f6f63cf7f8b418ac9ca04408c4579f4" + integrity sha512-38Y8f7YUhce/K7RMwTp7m0uCumpv9hZkitCbBClqQIow1qSbCvGkcegKOXpEWCQLfWmevgRiWokZ1GkpfhbZug== + dependencies: + regenerator-runtime "^0.13.4" + "@babel/template@^7.16.7", "@babel/template@^7.3.3": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.16.7.tgz#8d126c8701fde4d66b264b3eba3d96f07666d155" @@ -1514,6 +1521,13 @@ version "3.2.8" resolved "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz#8d53636d045e1776e2a2ec6613e57330dd9ce856" +"@matrix-org/react-sdk-module-api@^0.0.3": + version "0.0.3" + resolved "https://registry.yarnpkg.com/@matrix-org/react-sdk-module-api/-/react-sdk-module-api-0.0.3.tgz#a7ac1b18a72d18d08290b81fa33b0d8d00a77d2b" + integrity sha512-jQmLhVIanuX0g7Jx1OIqlzs0kp72PfSpv3umi55qVPYcAPQmO252AUs0vncatK8O4e013vohdnNhly19a/kmLQ== + dependencies: + "@babel/runtime" "^7.17.9" + "@mrmlnc/readdir-enhanced@^2.2.1": version "2.2.1" resolved "https://registry.yarnpkg.com/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz#524af240d1a360527b730475ecfa1344aa540dde" @@ -13436,6 +13450,11 @@ yaml@^1.10.0: resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== +yaml@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.0.1.tgz#71886d6021f3da28169dbefde78d4dd0f8d83650" + integrity sha512-1NpAYQ3wjzIlMs0mgdBmYzLkFgWBIWrzYVDYfrixhoFNNgJ444/jT2kUT2sicRbJES3oQYRZugjB6Ro8SjKeFg== + yargs-parser@^13.1.2: version "13.1.2" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-13.1.2.tgz#130f09702ebaeef2650d54ce6e3e5706f7a4fb38"