Fix UI test flakiness flakiness by tweaking our RootMatchers.

These tests have been flaky for months, and in the last couple weeks have felt worse than ever. This PR fixes them by: - Introducing `onWorkflowView`, which is like `Espresso.onView` but looks for views in _all_ roots, instead of using the default root matcher. This introduces a risk of ambiguous matches in new tests, but that should be obvious if we ever hit it, and we can reduce the root scope then. - Introducing `workflowPressBack` which is like `Espresso.pressBack` but looks only in the focused window, to make sure it doesn't end up finding a hidden window and sending the back event to the wrong view, which ends up looking like a freeze in the tests.
square · Feb 4, 2021 · b12a1db · b12a1db
1 parent 2c48954
commit b12a1db
Show file tree

Hide file tree

Showing 17 changed files with 125 additions and 82 deletions.
diff --git a/.buildscript/android-ui-tests.gradle b/.buildscript/android-ui-tests.gradle
@@ -10,6 +10,7 @@ android {
 }
 
 dependencies {
+  androidTestImplementation project(":workflow-ui:internal-testing-android")
   androidTestImplementation Deps.get("test.androidx.espresso.core")
   androidTestImplementation Deps.get("test.androidx.junitExt")
 }
diff --git a/...containers/app-poetry/src/androidTest/java/com/squareup/sample/poetryapp/PoetryAppTest.kt b/...containers/app-poetry/src/androidTest/java/com/squareup/sample/poetryapp/PoetryAppTest.kt
@@ -1,12 +1,12 @@
 package com.squareup.sample.poetryapp
 
-import androidx.test.espresso.Espresso.onView
 import androidx.test.espresso.assertion.ViewAssertions.matches
 import androidx.test.espresso.matcher.ViewMatchers.isDisplayed
 import androidx.test.espresso.matcher.ViewMatchers.withText
 import androidx.test.ext.junit.rules.ActivityScenarioRule
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import com.squareup.sample.container.poetryapp.R
+import com.squareup.workflow1.ui.internal.test.onWorkflowView
 import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -17,7 +17,7 @@ class PoetryAppTest {
   @Rule @JvmField val scenarioRule = ActivityScenarioRule(PoetryActivity::class.java)
 
   @Test fun launches() {
-    onView(withText(R.string.poems))
+    onWorkflowView(withText(R.string.poems))
         .check(matches(isDisplayed()))
   }
 }
diff --git a/...es/containers/app-raven/src/androidTest/java/com/squareup/sample/ravenapp/RavenAppTest.kt b/...es/containers/app-raven/src/androidTest/java/com/squareup/sample/ravenapp/RavenAppTest.kt
@@ -1,11 +1,11 @@
 package com.squareup.sample.ravenapp
 
-import androidx.test.espresso.Espresso.onView
 import androidx.test.espresso.assertion.ViewAssertions.matches
 import androidx.test.espresso.matcher.ViewMatchers.isDisplayed
 import androidx.test.espresso.matcher.ViewMatchers.withText
 import androidx.test.ext.junit.rules.ActivityScenarioRule
 import androidx.test.ext.junit.runners.AndroidJUnit4
+import com.squareup.workflow1.ui.internal.test.onWorkflowView
 import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -16,7 +16,7 @@ class RavenAppTest {
   @Rule @JvmField val scenarioRule = ActivityScenarioRule(RavenActivity::class.java)
 
   @Test fun launches() {
-    onView(withText("The Raven"))
+    onWorkflowView(withText("The Raven"))
         .check(matches(isDisplayed()))
   }
 }
diff --git a/...n/src/androidTest/java/com/squareup/sample/hellobackbutton/HelloBackButtonEspressoTest.kt b/...n/src/androidTest/java/com/squareup/sample/hellobackbutton/HelloBackButtonEspressoTest.kt
@@ -1,14 +1,14 @@
 package com.squareup.sample.hellobackbutton
 
-import androidx.test.espresso.Espresso.onView
-import androidx.test.espresso.Espresso.pressBack
 import androidx.test.espresso.action.ViewActions.click
 import androidx.test.espresso.assertion.ViewAssertions.matches
 import androidx.test.espresso.matcher.ViewMatchers.isDisplayed
 import androidx.test.espresso.matcher.ViewMatchers.withId
 import androidx.test.espresso.matcher.ViewMatchers.withText
 import androidx.test.ext.junit.rules.ActivityScenarioRule
 import androidx.test.ext.junit.runners.AndroidJUnit4
+import com.squareup.workflow1.ui.internal.test.onWorkflowView
+import com.squareup.workflow1.ui.internal.test.workflowPressBack
 import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -19,24 +19,24 @@ class HelloBackButtonEspressoTest {
   @Rule @JvmField val scenarioRule = ActivityScenarioRule(HelloBackButtonActivity::class.java)
 
   @Test fun wrappedTakesPrecedence() {
-    onView(withId(R.id.hello_message)).apply {
+    onWorkflowView(withId(R.id.hello_message)).apply {
       check(matches(withText("Able")))
       perform(click())
       check(matches(withText("Baker")))
       perform(click())
       check(matches(withText("Charlie")))
-      pressBack()
+      workflowPressBack()
       check(matches(withText("Baker")))
-      pressBack()
+      workflowPressBack()
       check(matches(withText("Able")))
     }
   }
 
   @Test fun outerHandlerAppliesIfWrappedHandlerIsNull() {
-    onView(withId(R.id.hello_message)).apply {
-      pressBack()
-      onView(withText("Are you sure you want to do this thing?"))
-          .check(matches(isDisplayed()))
+    onWorkflowView(withId(R.id.hello_message)).apply {
+      workflowPressBack()
+      onWorkflowView(withText("Are you sure you want to do this thing?"))
+        .check(matches(isDisplayed()))
     }
   }
 }
diff --git a/samples/dungeon/app/src/androidTest/java/com/squareup/sample/dungeon/DungeonAppTest.kt b/samples/dungeon/app/src/androidTest/java/com/squareup/sample/dungeon/DungeonAppTest.kt
@@ -1,11 +1,11 @@
 package com.squareup.sample.dungeon
 
-import androidx.test.espresso.Espresso.onView
 import androidx.test.espresso.assertion.ViewAssertions.matches
 import androidx.test.espresso.matcher.ViewMatchers.isDisplayed
 import androidx.test.espresso.matcher.ViewMatchers.withText
 import androidx.test.ext.junit.rules.ActivityScenarioRule
 import androidx.test.ext.junit.runners.AndroidJUnit4
+import com.squareup.workflow1.ui.internal.test.onWorkflowView
 import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -16,7 +16,7 @@ class DungeonAppTest {
   @Rule @JvmField val scenarioRule = ActivityScenarioRule(DungeonActivity::class.java)
 
   @Test fun loadsBoardsList() {
-    onView(withText(R.string.boards_list_label))
+   onWorkflowView(withText(R.string.boards_list_label))
         .check(matches(isDisplayed()))
   }
 }
diff --git a/...ndroidTest/java/com/squareup/sample/helloworkflowfragment/HelloWorkflowFragmentAppTest.kt b/...ndroidTest/java/com/squareup/sample/helloworkflowfragment/HelloWorkflowFragmentAppTest.kt
@@ -1,12 +1,12 @@
 package com.squareup.sample.helloworkflowfragment
 
-import androidx.test.espresso.Espresso.onView
 import androidx.test.espresso.action.ViewActions.click
 import androidx.test.espresso.assertion.ViewAssertions.matches
 import androidx.test.espresso.matcher.ViewMatchers.isDisplayed
 import androidx.test.espresso.matcher.ViewMatchers.withText
 import androidx.test.ext.junit.rules.ActivityScenarioRule
 import androidx.test.ext.junit.runners.AndroidJUnit4
+import com.squareup.workflow1.ui.internal.test.onWorkflowView
 import org.hamcrest.Matchers.containsString
 import org.junit.Rule
 import org.junit.Test
@@ -18,15 +18,15 @@ class HelloWorkflowFragmentAppTest {
   @Rule @JvmField val scenarioRule = ActivityScenarioRule(HelloWorkflowFragmentActivity::class.java)
 
   @Test fun togglesHelloAndGoodbye() {
-    onView(withText(containsString("Hello")))
+    onWorkflowView(withText(containsString("Hello")))
       .check(matches(isDisplayed()))
       .perform(click())
 
-    onView(withText(containsString("Goodbye")))
+    onWorkflowView(withText(containsString("Goodbye")))
       .check(matches(isDisplayed()))
       .perform(click())
 
-    onView(withText(containsString("Hello")))
+    onWorkflowView(withText(containsString("Hello")))
       .check(matches(isDisplayed()))
   }
 }
diff --git a/...o-workflow/src/androidTest/java/com/squareup/sample/helloworkflow/HelloWorkflowAppTest.kt b/...o-workflow/src/androidTest/java/com/squareup/sample/helloworkflow/HelloWorkflowAppTest.kt
@@ -1,12 +1,12 @@
 package com.squareup.sample.helloworkflow
 
-import androidx.test.espresso.Espresso.onView
 import androidx.test.espresso.action.ViewActions.click
 import androidx.test.espresso.assertion.ViewAssertions.matches
 import androidx.test.espresso.matcher.ViewMatchers.isDisplayed
 import androidx.test.espresso.matcher.ViewMatchers.withText
 import androidx.test.ext.junit.rules.ActivityScenarioRule
 import androidx.test.ext.junit.runners.AndroidJUnit4
+import com.squareup.workflow1.ui.internal.test.onWorkflowView
 import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -17,15 +17,15 @@ class HelloWorkflowAppTest {
   @Rule @JvmField val scenarioRule = ActivityScenarioRule(HelloWorkflowActivity::class.java)
 
   @Test fun togglesHelloAndGoodbye() {
-    onView(withText("Hello"))
+    onWorkflowView(withText("Hello"))
         .check(matches(isDisplayed()))
         .perform(click())
 
-    onView(withText("Goodbye"))
+    onWorkflowView(withText("Goodbye"))
         .check(matches(isDisplayed()))
         .perform(click())
 
-    onView(withText("Hello"))
+    onWorkflowView(withText("Hello"))
         .check(matches(isDisplayed()))
   }
 }
diff --git a/...sibility/src/androidTest/java/com/squareup/sample/stubvisibility/StubVisibilityAppTest.kt b/...sibility/src/androidTest/java/com/squareup/sample/stubvisibility/StubVisibilityAppTest.kt
@@ -1,13 +1,13 @@
 package com.squareup.sample.stubvisibility
 
-import androidx.test.espresso.Espresso.onView
 import androidx.test.espresso.action.ViewActions.click
 import androidx.test.espresso.assertion.ViewAssertions.matches
 import androidx.test.espresso.matcher.ViewMatchers.isDisplayed
 import androidx.test.espresso.matcher.ViewMatchers.withId
 import androidx.test.espresso.matcher.ViewMatchers.withText
 import androidx.test.ext.junit.rules.ActivityScenarioRule
 import androidx.test.ext.junit.runners.AndroidJUnit4
+import com.squareup.workflow1.ui.internal.test.onWorkflowView
 import org.hamcrest.CoreMatchers.not
 import org.junit.Rule
 import org.junit.Test
@@ -19,19 +19,19 @@ class StubVisibilityAppTest {
   @Rule @JvmField val scenarioRule = ActivityScenarioRule(StubVisibilityActivity::class.java)
 
   @Test fun togglesFooter() {
-    onView(withId(R.id.should_be_wrapped))
+    onWorkflowView(withId(R.id.should_be_wrapped))
         .check(matches(not(isDisplayed())))
 
-    onView(withText("Click to show footer"))
+    onWorkflowView(withText("Click to show footer"))
         .perform(click())
 
-    onView(withId(R.id.should_be_wrapped))
+    onWorkflowView(withId(R.id.should_be_wrapped))
         .check(matches(isDisplayed()))
 
-    onView(withText("Click to hide footer"))
+    onWorkflowView(withText("Click to hide footer"))
         .perform(click())
 
-    onView(withId(R.id.should_be_wrapped))
+    onWorkflowView(withId(R.id.should_be_wrapped))
         .check(matches(not(isDisplayed())))
   }
 }
diff --git a/samples/tictactoe/app/src/androidTest/java/com/squareup/sample/TicTacToeEspressoTest.kt b/samples/tictactoe/app/src/androidTest/java/com/squareup/sample/TicTacToeEspressoTest.kt
@@ -3,8 +3,6 @@ package com.squareup.sample
 import android.content.pm.ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE
 import android.content.pm.ActivityInfo.SCREEN_ORIENTATION_PORTRAIT
 import android.view.View
-import androidx.test.espresso.Espresso.onView
-import androidx.test.espresso.Espresso.pressBack
 import androidx.test.espresso.IdlingRegistry
 import androidx.test.espresso.ViewInteraction
 import androidx.test.espresso.action.ViewActions.click
@@ -27,6 +25,8 @@ import com.squareup.workflow1.ui.ViewEnvironment
 import com.squareup.workflow1.ui.WorkflowUiExperimentalApi
 import com.squareup.workflow1.ui.environment
 import com.squareup.workflow1.ui.getRendering
+import com.squareup.workflow1.ui.internal.test.onWorkflowView
+import com.squareup.workflow1.ui.internal.test.workflowPressBack
 import org.junit.After
 import org.junit.Before
 import org.junit.Rule
@@ -62,11 +62,11 @@ class TicTacToeEspressoTest {
     // Start a game so that there's something interesting in the Activity window.
     // (Prior screens are all in a dialog window.)
 
-    onView(withId(R.id.login_email)).type("foo@bar")
-    onView(withId(R.id.login_password)).type("password")
-    onView(withId(R.id.login_button)).perform(click())
+    onWorkflowView(withId(R.id.login_email)).type("foo@bar")
+    onWorkflowView(withId(R.id.login_password)).type("password")
+    onWorkflowView(withId(R.id.login_button)).perform(click())
 
-    onView(withId(R.id.start_game)).perform(click())
+    onWorkflowView(withId(R.id.start_game)).perform(click())
 
     val environment = AtomicReference<ViewEnvironment>()
 
@@ -90,7 +90,7 @@ class TicTacToeEspressoTest {
     // actually seem to be necessary, originally did everything synchronously in the
     // lambda above and it all worked just fine. But that seems like a land mine.)
 
-    onView(withId(R.id.game_play_toolbar))
+    onWorkflowView(withId(R.id.game_play_toolbar))
       .check(matches(hasDescendant(withText("O, place your ${Player.O.symbol}"))))
 
     // Now that we're confident the views have updated, back to the activity
@@ -105,76 +105,76 @@ class TicTacToeEspressoTest {
   }
 
   @Test fun configChangeReflectsWorkflowState() {
-    onView(withId(R.id.login_email)).type("bad email")
-    onView(withId(R.id.login_button)).perform(click())
+    onWorkflowView(withId(R.id.login_email)).type("bad email")
+    onWorkflowView(withId(R.id.login_button)).perform(click())
 
-    onView(withId(R.id.login_error_message)).check(matches(withText("Invalid address")))
+    onWorkflowView(withId(R.id.login_error_message)).check(matches(withText("Invalid address")))
     rotate()
-    onView(withId(R.id.login_error_message)).check(matches(withText("Invalid address")))
+    onWorkflowView(withId(R.id.login_error_message)).check(matches(withText("Invalid address")))
   }
 
   @Test fun editTextSurvivesConfigChange() {
-    onView(withId(R.id.login_email)).type("foo@bar")
-    onView(withId(R.id.login_password)).type("password")
+    onWorkflowView(withId(R.id.login_email)).type("foo@bar")
+    onWorkflowView(withId(R.id.login_password)).type("password")
     rotate()
-    onView(withId(R.id.login_email)).check(matches(withText("foo@bar")))
+    onWorkflowView(withId(R.id.login_email)).check(matches(withText("foo@bar")))
     // Don't save fields that shouldn't be.
-    onView(withId(R.id.login_password)).check(matches(withText("")))
+    onWorkflowView(withId(R.id.login_password)).check(matches(withText("")))
   }
 
   @Test fun backStackPopRestoresViewState() {
     // The loading screen is pushed onto the back stack.
-    onView(withId(R.id.login_email)).type("foo@bar")
-    onView(withId(R.id.login_password)).type("bad password")
-    onView(withId(R.id.login_button)).perform(click())
+    onWorkflowView(withId(R.id.login_email)).type("foo@bar")
+    onWorkflowView(withId(R.id.login_password)).type("bad password")
+    onWorkflowView(withId(R.id.login_button)).perform(click())
 
     // Loading ends with an error, and we pop back to login. The
     // email should have been restored from view state.
-    onView(withId(R.id.login_email)).check(matches(withText("foo@bar")))
-    onView(withId(R.id.login_error_message))
+    onWorkflowView(withId(R.id.login_email)).check(matches(withText("foo@bar")))
+    onWorkflowView(withId(R.id.login_error_message))
       .check(matches(withText("Unknown email or invalid password")))
   }
 
   @Test fun dialogSurvivesConfigChange() {
-    onView(withId(R.id.login_email)).type("foo@bar")
-    onView(withId(R.id.login_password)).type("password")
-    onView(withId(R.id.login_button)).perform(click())
+    onWorkflowView(withId(R.id.login_email)).type("foo@bar")
+    onWorkflowView(withId(R.id.login_password)).type("password")
+    onWorkflowView(withId(R.id.login_button)).perform(click())
 
-    onView(withId(R.id.player_X)).type("Mister X")
-    onView(withId(R.id.player_O)).type("Sister O")
-    onView(withId(R.id.start_game)).perform(click())
+    onWorkflowView(withId(R.id.player_X)).type("Mister X")
+    onWorkflowView(withId(R.id.player_O)).type("Sister O")
+    onWorkflowView(withId(R.id.start_game)).perform(click())
 
-    pressBack()
-    onView(withText("Do you really want to concede the game?")).check(matches(isDisplayed()))
+    workflowPressBack()
+    onWorkflowView(withText("Do you really want to concede the game?")).check(matches(isDisplayed()))
     rotate()
-    onView(withText("Do you really want to concede the game?")).check(matches(isDisplayed()))
+    onWorkflowView(withText("Do you really want to concede the game?")).check(matches(isDisplayed()))
   }
 
   @Test fun canGoBackInModalView() {
     // Log in and hit the 2fa screen.
-    onView(withId(R.id.login_email)).type("foo@2fa")
-    onView(withId(R.id.login_password)).type("password")
-    onView(withId(R.id.login_button)).perform(click())
-    onView(withId(R.id.second_factor)).check(matches(isDisplayed()))
+    onWorkflowView(withId(R.id.login_email)).type("foo@2fa")
+    onWorkflowView(withId(R.id.login_password)).type("password")
+    onWorkflowView(withId(R.id.login_button)).perform(click())
+    onWorkflowView(withId(R.id.second_factor)).check(matches(isDisplayed()))
 
     // Use the back button to go back and see the login screen again.
-    pressBack()
+    workflowPressBack()
     // Make sure edit text was restored from view state cached by the back stack container.
-    onView(withId(R.id.login_email)).check(matches(withText("foo@2fa")))
+    onWorkflowView(withId(R.id.login_email)).check(matches(withText("foo@2fa")))
   }
 
   @Test fun configChangePreservesBackStackViewStateCache() {
     // Log in and hit the 2fa screen.
-    onView(withId(R.id.login_email)).type("foo@2fa")
-    onView(withId(R.id.login_password)).type("password")
-    onView(withId(R.id.login_button)).perform(click())
-    onView(withId(R.id.second_factor)).check(matches(isDisplayed()))
+    onWorkflowView(withId(R.id.login_email)).type("foo@2fa")
+    onWorkflowView(withId(R.id.login_password)).type("password")
+    onWorkflowView(withId(R.id.login_button)).perform(click())
+    onWorkflowView(withId(R.id.second_factor)).check(matches(isDisplayed()))
 
     // Rotate and then use the back button to go back and see the login screen again.
     rotate()
-    pressBack()
+    workflowPressBack()
     // Make sure edit text was restored from view state cached by the back stack container.
-    onView(withId(R.id.login_email)).check(matches(withText("foo@2fa")))
+    onWorkflowView(withId(R.id.login_email)).check(matches(withText("foo@2fa")))
   }
 
   private fun ViewInteraction.type(text: String) {

diff --git a/samples/tictactoe/common/src/main/java/com/squareup/sample/mainworkflow/TicTacToeWorkflow.kt b/samples/tictactoe/common/src/main/java/com/squareup/sample/mainworkflow/TicTacToeWorkflow.kt
@@ -73,7 +73,7 @@ class TicTacToeWorkflow(
       if (panels.isEmpty()) {
         childRendering
       } else {
-        // To prompt for player names, the child puts up a panel — that is, a modal view
+        // To prompt for player names, the child puts up a panel — that is, a modal view
         // hosting a BackStackScreen. If they cancel that, we'd like a visual effect of
         // popping back to the auth flow in that same panel. To get this effect we run
         // an authWorkflow and put its BackStackScreen behind this one.