Merge pull request #4516 from vector-im/feature/adm/ui-test-ci-tweaks

UI test CI tweaks

.github/workflows/sanity_test.yml

@@ -56,10 +56,10 @@ jobs:
           java-version: '11'
       - name: Run sanity tests on API ${{ matrix.api-level }}
         uses: reactivecircus/android-emulator-runner@v2
-        continue-on-error: true # allow pipeline to upload failure results
         with:
           emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
           api-level: ${{ matrix.api-level }}
+          profile: 24 # Pixel 5
           emulator-build: 7425822  # workaround to emulator bug: https://github.com/ReactiveCircus/android-emulator-runner/issues/160
           script: |
             adb root
@@ -67,13 +67,12 @@ jobs:
             touch emulator.log
             chmod 777 emulator.log
             adb logcat >> emulator.log &
-            ./gradlew $CI_GRADLE_ARG_PROPERTIES -PallWarningsAsErrors=false connectedGplayDebugAndroidTest -Pandroid.testInstrumentationRunnerArguments.class=im.vector.app.ui.UiAllScreensSanityTest || adb pull storage/emulated/0/Pictures/failure_screenshots
-
+            ./gradlew $CI_GRADLE_ARG_PROPERTIES -PallWarningsAsErrors=false connectedGplayDebugAndroidTest -Pandroid.testInstrumentationRunnerArguments.class=im.vector.app.ui.UiAllScreensSanityTest || adb pull storage/emulated/0/Pictures/failure_screenshots && exit 1
       - name: Upload Failing Test Report Log
-        if: failure()
         uses: actions/upload-artifact@v2
+        if: failure()
         with:
           name: sanity-error-results
           path: |
             emulator.log
-            failure_screenshots/
+            failure_screenshots/

vector/src/androidTest/java/im/vector/app/EspressoExt.kt

@@ -68,6 +68,18 @@ object EspressoHelper {
     }
 }
 
+fun withRetry(attempts: Int = 3, action: () -> Unit) {
+    runCatching { action() }.onFailure {
+        val remainingAttempts = attempts - 1
+        if (remainingAttempts <= 0) {
+            throw it
+        } else {
+            Thread.sleep(500)
+            withRetry(remainingAttempts, action)
+        }
+    }
+}
+
 fun getString(@StringRes id: Int): String {
     return EspressoHelper.getCurrentActivity()!!.resources.getString(id)
 }
@@ -235,11 +247,16 @@ fun clickOnAndGoBack(@StringRes name: Int, block: () -> Unit) {
     Espresso.pressBack()
 }
 
-inline fun <reified T : VectorBaseBottomSheetDialogFragment<*>> interactWithSheet(contentMatcher: Matcher<View>, noinline block: () -> Unit = {}) {
+inline fun <reified T : VectorBaseBottomSheetDialogFragment<*>> interactWithSheet(
+        contentMatcher: Matcher<View>,
+        @BottomSheetBehavior.State openState: Int = BottomSheetBehavior.STATE_EXPANDED,
+        @BottomSheetBehavior.State exitState: Int = BottomSheetBehavior.STATE_HIDDEN,
+        noinline block: () -> Unit = {}
+) {
     waitUntilViewVisible(contentMatcher)
     val behaviour = (EspressoHelper.getBottomSheetDialog<T>()!!.dialog as BottomSheetDialog).behavior
-    withIdlingResource(BottomSheetResource(behaviour, BottomSheetBehavior.STATE_EXPANDED), block)
-    withIdlingResource(BottomSheetResource(behaviour, BottomSheetBehavior.STATE_HIDDEN)) {}
+    withIdlingResource(BottomSheetResource(behaviour, openState), block)
+    withIdlingResource(BottomSheetResource(behaviour, exitState)) {}
 }
 
 class BottomSheetResource(

vector/src/androidTest/java/im/vector/app/espresso/tools/WaitActivity.kt

@@ -18,17 +18,25 @@ package im.vector.app.espresso.tools
 
 import android.app.Activity
 import android.view.View
-import androidx.test.espresso.Espresso
+import androidx.test.espresso.Espresso.onView
+import androidx.test.espresso.assertion.ViewAssertions
+import androidx.test.espresso.matcher.RootMatchers.isDialog
 import androidx.test.espresso.matcher.ViewMatchers
 import im.vector.app.activityIdlingResource
 import im.vector.app.waitForView
 import im.vector.app.withIdlingResource
 import org.hamcrest.Matcher
+import org.hamcrest.Matchers.not
 
 inline fun <reified T : Activity> waitUntilActivityVisible(noinline block: (() -> Unit) = {}) {
     withIdlingResource(activityIdlingResource(T::class.java), block)
 }
 
 fun waitUntilViewVisible(viewMatcher: Matcher<View>) {
-    Espresso.onView(ViewMatchers.isRoot()).perform(waitForView(viewMatcher))
+    onView(ViewMatchers.isRoot()).perform(waitForView(viewMatcher))
+}
+
+fun waitUntilDialogVisible(viewMatcher: Matcher<View>) {
+    onView(viewMatcher).inRoot(isDialog()).check(ViewAssertions.matches(ViewMatchers.isDisplayed()))
+    waitUntilViewVisible(viewMatcher)
 }

vector/src/androidTest/java/im/vector/app/ui/UiAllScreensSanityTest.kt

@@ -16,6 +16,7 @@
 
 package im.vector.app.ui
 
+import androidx.test.espresso.IdlingPolicies
 import androidx.test.ext.junit.rules.ActivityScenarioRule
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import androidx.test.filters.LargeTest
@@ -30,6 +31,7 @@ import org.junit.Test
 import org.junit.rules.RuleChain
 import org.junit.runner.RunWith
 import java.util.UUID
+import java.util.concurrent.TimeUnit
 
 /**
  * This test aim to open every possible screen of the application
@@ -51,6 +53,8 @@ class UiAllScreensSanityTest {
     // 2021-04-08 Testing 429 change
     @Test
     fun allScreensTest() {
+        IdlingPolicies.setMasterPolicyTimeout(120, TimeUnit.SECONDS)
+
         // Create an account
         val userId = "UiTest_" + UUID.randomUUID().toString()
         elementRobot.signUp(userId)

vector/src/androidTest/java/im/vector/app/ui/robot/ElementRobot.kt

@@ -28,6 +28,7 @@ import com.adevinta.android.barista.interaction.BaristaDrawerInteractions.openDr
 import im.vector.app.EspressoHelper
 import im.vector.app.R
 import im.vector.app.espresso.tools.waitUntilActivityVisible
+import im.vector.app.espresso.tools.waitUntilDialogVisible
 import im.vector.app.espresso.tools.waitUntilViewVisible
 import im.vector.app.features.createdirect.CreateDirectRoomActivity
 import im.vector.app.features.home.HomeActivity
@@ -104,17 +105,19 @@ class ElementRobot {
         }.isSuccess
 
         if (expectSignOutWarning != isShowingSignOutWarning) {
-            Timber.w("Unexpected sign out flow, expected warning to be: ${expectSignOutWarning.toWarningType()} but was ${isShowingSignOutWarning.toWarningType()}")
+            val expected = expectSignOutWarning.toWarningType()
+            val actual = isShowingSignOutWarning.toWarningType()
+            Timber.w("Unexpected sign out flow, expected warning to be: $expected but was $actual")
         }
 
         if (isShowingSignOutWarning) {
             // We have sent a message in a e2e room, accept to loose it
             clickOn(R.id.exitAnywayButton)
             // Dark pattern
-            waitUntilViewVisible(withId(android.R.id.button2))
+            waitUntilDialogVisible(withId(android.R.id.button2))
             clickDialogNegativeButton()
         } else {
-            waitUntilViewVisible(withId(android.R.id.button1))
+            waitUntilDialogVisible(withId(android.R.id.button1))
             clickDialogPositiveButton()
         }
 

vector/src/androidTest/java/im/vector/app/ui/robot/MessageMenuRobot.kt

@@ -17,9 +17,13 @@
 package im.vector.app.ui.robot
 
 import androidx.test.espresso.Espresso.pressBack
+import androidx.test.espresso.matcher.ViewMatchers.withText
 import com.adevinta.android.barista.interaction.BaristaClickInteractions.clickOn
 import com.adevinta.android.barista.interaction.BaristaListInteractions.clickListItem
+import com.google.android.material.bottomsheet.BottomSheetBehavior
 import im.vector.app.R
+import im.vector.app.features.home.room.detail.timeline.edithistory.ViewEditHistoryBottomSheet
+import im.vector.app.interactWithSheet
 import java.lang.Thread.sleep
 
 class MessageMenuRobot(
@@ -36,7 +40,9 @@ class MessageMenuRobot(
 
     fun editHistory() {
         clickOn(R.string.message_view_edit_history)
-        pressBack()
+        interactWithSheet<ViewEditHistoryBottomSheet>(withText(R.string.message_edits), openState = BottomSheetBehavior.STATE_COLLAPSED) {
+            pressBack()
+        }
         autoClosed = true
     }
 

vector/src/androidTest/java/im/vector/app/ui/robot/OnboardingRobot.kt

@@ -26,6 +26,7 @@ import com.adevinta.android.barista.assertion.BaristaVisibilityAssertions.assert
 import com.adevinta.android.barista.interaction.BaristaClickInteractions.clickOn
 import com.adevinta.android.barista.interaction.BaristaEditTextInteractions.writeTo
 import im.vector.app.R
+import im.vector.app.espresso.tools.waitUntilViewVisible
 import im.vector.app.waitForView
 
 class OnboardingRobot {
@@ -42,6 +43,7 @@ class OnboardingRobot {
                             userId: String,
                             password: String,
                             homeServerUrl: String) {
+        waitUntilViewVisible(withId(R.id.loginSplashSubmit))
         assertDisplayed(R.id.loginSplashSubmit, R.string.login_splash_submit)
         clickOn(R.id.loginSplashSubmit)
         assertDisplayed(R.id.loginServerTitle, R.string.login_server_title)

vector/src/androidTest/java/im/vector/app/ui/robot/RoomDetailRobot.kt

@@ -30,12 +30,15 @@ import com.adevinta.android.barista.interaction.BaristaClickInteractions.longCli
 import com.adevinta.android.barista.interaction.BaristaEditTextInteractions.writeTo
 import com.adevinta.android.barista.interaction.BaristaMenuClickInteractions.clickMenu
 import com.adevinta.android.barista.interaction.BaristaMenuClickInteractions.openMenu
+import com.google.android.material.bottomsheet.BottomSheetBehavior
 import im.vector.app.R
 import im.vector.app.espresso.tools.waitUntilViewVisible
 import im.vector.app.features.home.room.detail.timeline.action.MessageActionsBottomSheet
+import im.vector.app.features.home.room.detail.timeline.reactions.ViewReactionsBottomSheet
 import im.vector.app.features.reactions.data.EmojiDataSource
 import im.vector.app.interactWithSheet
 import im.vector.app.waitForView
+import im.vector.app.withRetry
 import java.lang.Thread.sleep
 
 class RoomDetailRobot {
@@ -68,10 +71,14 @@ class RoomDetailRobot {
         openMessageMenu(message) {
             addQuickReaction(quickReaction)
         }
+        println("Open reactions bottom sheet")
         // Open reactions
-        longClickOn(quickReaction)
+        longClickReaction(quickReaction)
         // wait for bottom sheet
-        pressBack()
+        interactWithSheet<ViewReactionsBottomSheet>(withText(R.string.reactions), openState = BottomSheetBehavior.STATE_COLLAPSED) {
+            pressBack()
+        }
+        println("Room Detail Robot: Open reaction from emoji picker")
         // Test add reaction
         openMessageMenu(message) {
             addReactionFromEmojiPicker()
@@ -81,16 +88,24 @@ class RoomDetailRobot {
             edit()
         }
         // TODO Cancel action
-        writeTo(R.id.composerEditText, "Hello universe!")
+        val edit = "Hello universe - long message to avoid espresso tapping edited!"
+        writeTo(R.id.composerEditText, edit)
         // Wait a bit for the keyboard layout to update
         waitUntilViewVisible(withId(R.id.sendButton))
         clickOn(R.id.sendButton)
         // Wait for the UI to update
-        waitUntilViewVisible(withText("Hello universe! (edited)"))
+        waitUntilViewVisible(withText("$edit (edited)"))
         // Open edit history
-        openMessageMenu("Hello universe! (edited)") {
+        openMessageMenu("$edit (edited)") {
             editHistory()
         }
+        waitUntilViewVisible(withId(R.id.composerEditText))
+    }
+
+    private fun longClickReaction(quickReaction: String) {
+        withRetry {
+            longClickOn(quickReaction)
+        }
     }
 
     fun openMessageMenu(message: String, block: MessageMenuRobot.() -> Unit) {
@@ -111,7 +126,7 @@ class RoomDetailRobot {
     }
 
     fun openSettings(block: RoomSettingsRobot.() -> Unit) {
-        clickOn(R.id.roomToolbarTitleView)
+        clickMenu(R.id.timeline_setting)
         waitForView(withId(R.id.roomProfileAvatarView))
         sleep(1000)
         block(RoomSettingsRobot())

vector/src/androidTest/java/im/vector/app/ui/robot/RoomSettingsRobot.kt

@@ -26,6 +26,7 @@ import com.adevinta.android.barista.interaction.BaristaDialogInteractions.clickD
 import com.adevinta.android.barista.interaction.BaristaListInteractions.clickListItem
 import im.vector.app.R
 import im.vector.app.espresso.tools.waitUntilActivityVisible
+import im.vector.app.espresso.tools.waitUntilDialogVisible
 import im.vector.app.espresso.tools.waitUntilViewVisible
 import im.vector.app.features.roommemberprofile.RoomMemberProfileActivity
 
@@ -78,9 +79,9 @@ class RoomSettingsRobot {
 
         // Room permissions
         clickListItem(R.id.matrixProfileRecyclerView, 17)
-        waitUntilViewVisible(withText(R.string.room_permissions_title))
+        waitUntilViewVisible(withText(R.string.room_permissions_change_room_avatar))
         clickOn(R.string.room_permissions_change_room_avatar)
-        waitUntilViewVisible(withId(android.R.id.button2))
+        waitUntilDialogVisible(withId(android.R.id.button2))
         clickDialogNegativeButton()
         waitUntilViewVisible(withText(R.string.room_permissions_title))
         // Toggle
@@ -95,7 +96,7 @@ class RoomSettingsRobot {
 
     private fun leaveRoom(block: DialogRobot.() -> Unit) {
         clickListItem(R.id.matrixProfileRecyclerView, 13)
-        waitUntilViewVisible(withId(android.R.id.button2))
+        waitUntilDialogVisible(withId(android.R.id.button2))
         val dialogRobot = DialogRobot()
         block(dialogRobot)
         if (dialogRobot.returnedToPreviousScreen) {
@@ -135,7 +136,7 @@ class RoomSettingsRobot {
 
         // Role
         clickListItem(R.id.matrixProfileRecyclerView, 3)
-        waitUntilViewVisible(withId(android.R.id.button2))
+        waitUntilDialogVisible(withId(android.R.id.button2))
         clickDialogNegativeButton()
         waitUntilViewVisible(withId(R.id.matrixProfileRecyclerView))
         pressBack()