Merge pull request #7128 from vector-im/feature/adm/screenshot-testing-setup

Paparazzi screenshot testing setup

.gitattributes

@@ -0,0 +1 @@
+**/snapshots/**/*.png filter=lfs diff=lfs merge=lfs -text

.github/workflows/tests.yml

@@ -24,12 +24,26 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
+          lfs: true
           fetch-depth: 0
       - uses: actions/setup-java@v3
         with:
           distribution: 'adopt'
           java-version: '11'
       - uses: gradle/gradle-build-action@v2
+
+      - name: Run screenshot tests
+        run: ./gradlew verifyScreenshots
+
+      - name: Archive Screenshot Results on Error
+        if: failure()
+        uses: actions/upload-artifact@v1
+        with:
+          name: screenshot-results
+          path: |
+            **/out/failures/
+            **/build/reports/tests/*UnitTest/
+
       - uses: actions/setup-python@v4
         with:
           python-version: 3.8

.github/workflows/validate-lfs.yml

@@ -0,0 +1,15 @@
+name: Validate Git LFS
+
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Validate
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          lfs: 'true'
+
+      - run: |
+          ./tools/validate_lfs.sh

.gitignore

@@ -22,3 +22,4 @@
 /package.json
 /yarn.lock
 /node_modules
+**/out/failures

build.gradle

@@ -33,6 +33,7 @@ buildscript {
         classpath "org.jetbrains.dokka:dokka-gradle-plugin:1.7.10"
         classpath "org.jetbrains.kotlinx:kotlinx-knit:0.4.0"
         classpath 'com.jakewharton:butterknife-gradle-plugin:10.2.3'
+        classpath 'app.cash.paparazzi:paparazzi-gradle-plugin:1.0.0'
         // NOTE: Do not place your application dependencies here; they belong
         // in the individual module build.gradle files
     }
@@ -295,3 +296,28 @@ dependencyAnalysis {
         }
     }
 }
+
+tasks.register("recordScreenshots", GradleBuild) {
+    startParameter.projectProperties.screenshot = ""
+    tasks = [':vector:recordPaparazziDebug']
+}
+
+tasks.register("verifyScreenshots", GradleBuild) {
+    startParameter.projectProperties.screenshot = ""
+    tasks = [':vector:verifyPaparazziDebug']
+}
+
+ext.initScreenshotTests = { project ->
+    def hasScreenshots = project.hasProperty("screenshot")
+    if (hasScreenshots) {
+        project.apply plugin: 'app.cash.paparazzi'
+    }
+    project.android.testOptions.unitTests.all {
+        def screenshotTestCapture = "**/*ScreenshotTest*"
+        if (hasScreenshots) {
+            include screenshotTestCapture
+        } else {
+            exclude screenshotTestCapture
+        }
+    }
+}

changelog.d/5798.misc

@@ -0,0 +1 @@
+Adds screenshot testing tooling

dependencies_groups.gradle

@@ -49,6 +49,7 @@ ext.groups = [
                 regex: [
                 ],
                 group: [
+                        'app.cash.paparazzi',
                         'ch.qos.logback',
                         'com.adevinta.android',
                         'com.airbnb.android',
@@ -150,11 +151,14 @@ ext.groups = [
                         'it.unimi.dsi',
                         'jakarta.activation',
                         'jakarta.xml.bind',
+                        'javax.activation',
                         'javax.annotation',
                         'javax.inject',
+                        'javax.xml.bind',
                         'jline',
                         'jp.wasabeef',
                         'junit',
+                        'kxml2',
                         'me.saket',
                         'net.bytebuddy',
                         'net.java',
@@ -183,11 +187,13 @@ ext.groups = [
                         'org.hamcrest',
                         'org.jacoco',
                         'org.java-websocket',
+                        'org.jcodec',
                         'org.jetbrains',
                         'org.jetbrains.dokka',
                         'org.jetbrains.intellij.deps',
                         'org.jetbrains.kotlin',
                         'org.jetbrains.kotlinx',
+                        'org.jetbrains.trove4j',
                         'org.json',
                         'org.jsoup',
                         'org.junit',

docs/screenshot_testing.md

@@ -0,0 +1,72 @@
+# Screenshot testing
+
+<!--- TOC -->
+
+* [Overview](#overview)
+* [Setup](#setup)
+* [Recording](#recording)
+* [Verifying](#verifying)
+* [Contributing](#contributing)
+* [Example](#example)
+
+<!--- END -->
+
+## Overview
+
+- Screenshot tests are tests which record the content of a rendered screen and verify subsequent runs to check if the screen renders differently.
+- Element uses [Paparazzi](https://github.com/cashapp/paparazzi) to render, record and verify android layouts. 
+- The screenshot verification occurs on every pull request as part of the `tests.yml` workflow.
+
+## Setup
+
+- Install Git LFS through your package manager of choice (`brew install git-lfs` | `yay -S git-lfs`).
+- Install the Git LFS hooks into the project.
+
+```bash
+# with element-android as the current working directory
+git lfs install --local
+```
+
+- If installed correctly, `git push` and `git pull` will now include LFS content.
+
+## Recording
+
+- `./gradlew recordScreenshots`
+- Paparazzi will generate images in `${module}/src/test/snapshots`, which will need to be committed to the repository using Git LFS.
+
+## Verifying
+
+- `./gradlew verifyScreenshots`
+- In the case of failure, Paparazzi will generate images in `${module}/out/failure`. The images will show the expected and actual screenshots along with a delta of the two images.
+
+## Contributing
+
+- When creating a test, the file (and class) name names must include `ScreenshotTest`, eg `ItemScreenshotTest`.
+- After creating the new test, record and commit the newly rendered screens.
+- `./tools/validate_lfs` can be ran to ensure everything is working correctly with Git LFS, the CI also runs this check.
+
+## Example
+
+```kotlin
+class PaparazziExampleScreenshotTest {
+
+    @get:Rule
+    val paparazzi = Paparazzi(
+            deviceConfig = PIXEL_3,
+            theme = "Theme.Vector.Light",
+    )
+
+    @Test
+    fun `example paparazzi test`() {
+        // Inflate the layout
+        val view = paparazzi.inflate<ConstraintLayout>(R.layout.item_radio)
+
+        // Bind data to the view
+        view.findViewById<TextView>(R.id.actionTitle).text = paparazzi.resources.getString(R.string.room_settings_all_messages)
+        view.findViewById<ImageView>(R.id.radioIcon).setImageResource(R.drawable.ic_radio_on)
+
+        // Record the bound view
+        paparazzi.snapshot(view)
+    }
+}
+```

gradle.properties

@@ -15,7 +15,7 @@ org.gradle.vfs.watch=true
 org.gradle.caching=true
 
 # Android Settings
-android.enableJetifier=true
+android.enableJetifier=false
 android.useAndroidX=true
 
 #Project Settings

tools/validate_lfs.sh

@@ -0,0 +1,29 @@
+#! /bin/bash
+
+#
+# Copyright (c) 2022 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Based on https://cashapp.github.io/paparazzi/#git-lfs
+
+# Compare the output of `git ls-files ':(attr:filter=lfs)'` against `git lfs ls-files`
+# If there's no diff we assume the files have been committed using git lfs
+diff <(git ls-files ':(attr:filter=lfs)' | sort) <(git lfs ls-files -n | sort) >/dev/null
+
+ret=$?
+if [[ $ret -ne 0 ]]; then
+  echo >&2 "Detected files committed without using Git LFS."
+  echo >&2 "Install git lfs (eg brew install git-lfs) and run 'git lfs install --local' within the root repository directory and re-commit your files."
+  exit 1
+fi

vector/build.gradle

@@ -24,6 +24,8 @@ project.android.buildTypes.all { buildType ->
             ]
 }
 
+initScreenshotTests(project)
+
 android {
     // Due to a bug introduced in Android gradle plugin 3.6.0, we have to specify the ndk version to use
     // Ref: https://issuetracker.google.com/issues/144111441