Merge pull request #1493 from matrix-org/dbkr/geni18n

Script to generate the translations base file

package.json

@@ -101,7 +101,9 @@
     "eslint-plugin-babel": "^4.0.1",
     "eslint-plugin-flowtype": "^2.30.0",
     "eslint-plugin-react": "^7.4.0",
+    "estree-walker": "^0.5.0",
     "expect": "^1.16.0",
+    "flow-parser": "^0.57.3",
     "json-loader": "^0.5.3",
     "karma": "^1.7.0",
     "karma-chrome-launcher": "^0.2.3",
@@ -121,6 +123,7 @@
     "rimraf": "^2.4.3",
     "sinon": "^1.17.3",
     "source-map-loader": "^0.1.5",
+    "walk": "^2.3.9",
     "webpack": "^1.12.14"
   }
 }

scripts/gen-i18n.js

@@ -0,0 +1,153 @@
+/*
+Copyright 2017 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * Regenerates the translations en_EN file by walking the source tree and
+ * parsing each file with flow-parser. Emits a JSON file with the
+ * translatable strings mapped to themselves in the order they appeared
+ * in the files and grouped by the file they appeared in.
+ *
+ * Usage: node scripts/gen-i18n.js
+ */
+const fs = require('fs');
+const path = require('path');
+
+const walk = require('walk');
+
+const flowParser = require('flow-parser');
+const estreeWalker = require('estree-walker');
+
+const TRANSLATIONS_FUNCS = ['_t', '_td', '_tJsx'];
+
+const INPUT_TRANSLATIONS_FILE = 'src/i18n/strings/en_EN.json';
+
+const FLOW_PARSER_OPTS = {
+  esproposal_class_instance_fields: true,
+  esproposal_class_static_fields: true,
+  esproposal_decorators: true,
+  esproposal_export_star_as: true,
+  types: true,
+};
+
+function getObjectValue(obj, key) {
+    for (const prop of obj.properties) {
+        if (prop.key.type == 'Identifier' && prop.key.name == key) {
+            return prop.value;
+        }
+    }
+    return null;
+}
+
+function getTKey(arg) {
+    if (arg.type == 'Literal') {
+        return arg.value;
+    } else if (arg.type == 'BinaryExpression' && arg.operator == '+') {
+        return getTKey(arg.left) + getTKey(arg.right);
+    } else if (arg.type == 'TemplateLiteral') {
+        return arg.quasis.map((q) => {
+            return q.value.raw;
+        }).join('');
+    }
+    return null;
+}
+
+function getTranslations(file) {
+    const tree = flowParser.parse(fs.readFileSync(file, { encoding: 'utf8' }), FLOW_PARSER_OPTS);
+
+    const trs = new Set();
+
+    estreeWalker.walk(tree, {
+        enter: function(node, parent) {
+            if (
+                node.type == 'CallExpression' &&
+                TRANSLATIONS_FUNCS.includes(node.callee.name)
+            ) {
+                const tKey = getTKey(node.arguments[0]);
+                // This happens whenever we call _t with non-literals (ie. whenever we've
+                // had to use a _td to compensate) so is expected.
+                if (tKey === null) return;
+
+                let isPlural = false;
+                if (node.arguments.length > 1 && node.arguments[1].type == 'ObjectExpression') {
+                    const countVal = getObjectValue(node.arguments[1], 'count');
+                    if (countVal) {
+                        isPlural = true;
+                    }
+                }
+
+                if (isPlural) {
+                    trs.add(tKey + "|other");
+                    const plurals = enPlurals[tKey];
+                    if (plurals) {
+                        for (const pluralType of Object.keys(plurals)) {
+                            trs.add(tKey + "|" + pluralType);
+                        }
+                    }
+                } else {
+                    trs.add(tKey);
+                }
+            }
+        }
+    });
+
+    return trs;
+}
+
+// gather en_EN plural strings from the input translations file:
+// the en_EN strings are all in the source with the exception of
+// pluralised strings, which we need to pull in from elsewhere.
+const inputTranslationsRaw = JSON.parse(fs.readFileSync(INPUT_TRANSLATIONS_FILE, { encoding: 'utf8' }));
+const enPlurals = {};
+
+for (const key of Object.keys(inputTranslationsRaw)) {
+    const parts = key.split("|");
+    if (parts.length > 1) {
+        const plurals = enPlurals[parts[0]] || {};
+        plurals[parts[1]] = inputTranslationsRaw[key];
+        enPlurals[parts[0]] = plurals;
+    }
+}
+
+const translatables = new Set();
+
+walk.walkSync("src", {
+    listeners: {
+        file: function(root, fileStats, next) {
+            if (!fileStats.name.endsWith('.js')) return;
+
+            const fullPath = path.join(root, fileStats.name);
+            const trs = getTranslations(fullPath);
+            console.log(`${fullPath} (${trs.size} strings)`);
+            for (const tr of trs.values()) {
+                translatables.add(tr);
+            }
+        },
+    }
+});
+
+const trObj = {};
+for (const tr of translatables) {
+    trObj[tr] = tr;
+    if (tr.includes("|")) {
+        trObj[tr] = inputTranslationsRaw[tr];
+    }
+}
+
+fs.writeFileSync(
+    "src/i18n/strings/en_EN.json",
+    JSON.stringify(trObj, translatables.values(), 4) + "\n"
+);
+