diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml
index e918466e6f..af55fe8cb4 100644
--- a/.buildkite/pipeline.yaml
+++ b/.buildkite/pipeline.yaml
@@ -85,6 +85,16 @@ steps:
           image: "node:10"
           propagate-environment: true
 
+  - label: "🌐 i18n"
+    command:
+      - "echo '--- Fetching Dependencies'"
+      - "yarn install"
+      - "echo '+++ Testing i18n output'"
+      - "yarn diff-i18n"
+    plugins:
+      - docker#v3.0.1:
+          image: "node:10"
+
   - wait
  
   - label: "🐴 Trigger riot-web"
diff --git a/package.json b/package.json
index 006fb10138..b718b4c13e 100644
--- a/package.json
+++ b/package.json
@@ -40,6 +40,7 @@
     "rethemendex": "res/css/rethemendex.sh",
     "i18n": "matrix-gen-i18n",
     "prunei18n": "matrix-prune-i18n",
+    "diff-i18n": "cp src/i18n/strings/en_EN.json src/i18n/strings/en_EN_orig.json && ./scripts/gen-i18n.js && node scripts/compare-file.js src/i18n/strings/en_EN_orig.json src/i18n/strings/en_EN.json",
     "build": "yarn reskindex && yarn start:init",
     "build:watch": "babel src -w --skip-initial-build -d lib --source-maps --copy-files",
     "emoji-data-strip": "node scripts/emoji-data-strip.js",
diff --git a/scripts/compare-file.js b/scripts/compare-file.js
new file mode 100644
index 0000000000..f53275ebfa
--- /dev/null
+++ b/scripts/compare-file.js
@@ -0,0 +1,10 @@
+const fs = require("fs");
+
+if (process.argv.length < 4) throw new Error("Missing source and target file arguments");
+
+const sourceFile = fs.readFileSync(process.argv[2], 'utf8');
+const targetFile = fs.readFileSync(process.argv[3], 'utf8');
+
+if (sourceFile !== targetFile) {
+    throw new Error("Files do not match");
+}