Merge pull request #102 from fknives/issue#97-compose-flakiness

Issue#97 Attempt to fix flakiness in Compose

app/src/androidTest/java/org/fnives/test/showcase/ui/AuthComposeInstrumentedTest.kt

@@ -1,11 +1,16 @@
 package org.fnives.test.showcase.ui
 
+import androidx.compose.ui.test.MainTestClock
 import androidx.compose.ui.test.junit4.StateRestorationTester
 import androidx.compose.ui.test.junit4.createComposeRule
+import androidx.test.espresso.IdlingRegistry
+import androidx.test.espresso.IdlingResource
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import org.fnives.test.showcase.R
 import org.fnives.test.showcase.android.testutil.screenshot.ScreenshotRule
-import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.anyResourceIdling
+import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.anyResourceNotIdle
+import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.awaitUntilIdle
+import org.fnives.test.showcase.android.testutil.synchronization.loopMainThreadFor
 import org.fnives.test.showcase.compose.screen.AppNavigation
 import org.fnives.test.showcase.core.integration.fake.FakeUserDataLocalStorage
 import org.fnives.test.showcase.core.login.IsUserLoggedInUseCase
@@ -18,6 +23,7 @@ import org.junit.Test
 import org.junit.rules.RuleChain
 import org.junit.runner.RunWith
 import org.koin.test.KoinTest
+import java.util.concurrent.Executors
 
 @RunWith(AndroidJUnit4::class)
 class AuthComposeInstrumentedTest : KoinTest {
@@ -53,8 +59,8 @@ class AuthComposeInstrumentedTest : KoinTest {
         mockServerScenarioSetup.setScenario(
             AuthScenario.Success(password = "alma", username = "banan")
         )
+        composeTestRule.mainClock.advanceTimeBy(SPLASH_DELAY)
 
-        composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
         navigationRobot.assertAuthScreen()
         robot.setPassword("alma")
             .setUsername("banan")
@@ -67,21 +73,21 @@ class AuthComposeInstrumentedTest : KoinTest {
         robot.assertLoading()
         composeTestRule.mainClock.autoAdvance = true
 
-        composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+        composeTestRule.mainClock.awaitIdlingResources()
         navigationRobot.assertHomeScreen()
     }
 
     /** GIVEN empty password and username WHEN signIn THEN error password is shown */
     @Test
     fun emptyPasswordShowsProperErrorMessage() {
-        composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+        composeTestRule.mainClock.advanceTimeBy(SPLASH_DELAY)
         navigationRobot.assertAuthScreen()
 
         robot.setUsername("banan")
             .assertUsername("banan")
             .clickOnLogin()
 
-        composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+        composeTestRule.mainClock.awaitIdlingResources()
         robot.assertErrorIsShown(R.string.password_is_invalid)
             .assertNotLoading()
         navigationRobot.assertAuthScreen()
@@ -90,7 +96,7 @@ class AuthComposeInstrumentedTest : KoinTest {
     /** GIVEN password and empty username WHEN signIn THEN error username is shown */
     @Test
     fun emptyUserNameShowsProperErrorMessage() {
-        composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+        composeTestRule.mainClock.advanceTimeBy(SPLASH_DELAY)
         navigationRobot.assertAuthScreen()
 
         robot
@@ -98,7 +104,7 @@ class AuthComposeInstrumentedTest : KoinTest {
             .assertPassword("banan")
             .clickOnLogin()
 
-        composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+        composeTestRule.mainClock.awaitIdlingResources()
         robot.assertErrorIsShown(R.string.username_is_invalid)
             .assertNotLoading()
         navigationRobot.assertAuthScreen()
@@ -111,7 +117,7 @@ class AuthComposeInstrumentedTest : KoinTest {
             AuthScenario.InvalidCredentials(password = "alma", username = "banan")
         )
 
-        composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+        composeTestRule.mainClock.advanceTimeBy(SPLASH_DELAY)
         navigationRobot.assertAuthScreen()
         robot.setUsername("alma")
             .setPassword("banan")
@@ -124,7 +130,7 @@ class AuthComposeInstrumentedTest : KoinTest {
         robot.assertLoading()
         composeTestRule.mainClock.autoAdvance = true
 
-        composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+        composeTestRule.mainClock.awaitIdlingResources()
         robot.assertErrorIsShown(R.string.credentials_invalid)
             .assertNotLoading()
         navigationRobot.assertAuthScreen()
@@ -137,7 +143,7 @@ class AuthComposeInstrumentedTest : KoinTest {
             AuthScenario.GenericError(username = "alma", password = "banan")
         )
 
-        composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+        composeTestRule.mainClock.advanceTimeBy(SPLASH_DELAY)
         navigationRobot.assertAuthScreen()
         robot.setUsername("alma")
             .setPassword("banan")
@@ -150,7 +156,7 @@ class AuthComposeInstrumentedTest : KoinTest {
         robot.assertLoading()
         composeTestRule.mainClock.autoAdvance = true
 
-        composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+        composeTestRule.mainClock.awaitIdlingResources()
         robot.assertErrorIsShown(R.string.something_went_wrong)
             .assertNotLoading()
         navigationRobot.assertAuthScreen()
@@ -159,7 +165,7 @@ class AuthComposeInstrumentedTest : KoinTest {
     /** GIVEN username and password WHEN restoring THEN username and password fields contain the same text */
     @Test
     fun restoringContentShowPreviousCredentials() {
-        composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+        composeTestRule.mainClock.advanceTimeBy(SPLASH_DELAY)
         navigationRobot.assertAuthScreen()
         robot.setUsername("alma")
             .setPassword("banan")
@@ -172,4 +178,36 @@ class AuthComposeInstrumentedTest : KoinTest {
         robot.assertUsername("alma")
             .assertPassword("banan")
     }
+
+    companion object {
+        private const val SPLASH_DELAY = 600L
+
+        // workaround, issue with idlingResources is tracked here https://github.com/robolectric/robolectric/issues/4807
+        /**
+         * Await the idling resource on a different thread while looping main.
+         */
+        fun MainTestClock.awaitIdlingResources() {
+            val idlingRegistry = IdlingRegistry.getInstance()
+            if (!anyResourceNotIdle()) return
+
+            val executor = Executors.newSingleThreadExecutor()
+            var isIdle = false
+            executor.submit {
+                do {
+                    idlingRegistry.resources
+                        .filterNot(IdlingResource::isIdleNow)
+                        .forEach { idlingResource ->
+                            idlingResource.awaitUntilIdle()
+                        }
+                } while (!idlingRegistry.resources.all(IdlingResource::isIdleNow))
+                isIdle = true
+            }
+            while (!isIdle) {
+                loopMainThreadFor(200L)
+            }
+            executor.shutdown()
+
+            advanceTimeByFrame()
+        }
+    }
 }

app/src/sharedTest/java/org/fnives/test/showcase/testutils/idling/DatabaseDispatcherTestRule.kt

@@ -3,7 +3,7 @@ package org.fnives.test.showcase.testutils.idling
 import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.test.StandardTestDispatcher
 import kotlinx.coroutines.test.TestDispatcher
-import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.anyResourceIdling
+import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.anyResourceNotIdle
 import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.awaitIdlingResources
 import org.fnives.test.showcase.android.testutil.synchronization.runOnUIAwaitOnCurrent
 import org.fnives.test.showcase.testutils.storage.TestDatabaseInitialization
@@ -42,7 +42,7 @@ class DatabaseDispatcherTestRule : TestRule {
     companion object {
         fun TestDispatcher.advanceUntilIdleWithIdlingResources() {
             scheduler.advanceUntilIdle() // advance until a request is sent
-            while (anyResourceIdling()) { // check if any request is in progress
+            while (anyResourceNotIdle()) { // check if any request is in progress
                 awaitIdlingResources() // complete all requests and other idling resources
                 scheduler.advanceUntilIdle() // run coroutines after request is finished
             }

codekata/compose.instructionset.md

@@ -159,7 +159,7 @@ private val mockServerScenarioSetup get() = mockServerScenarioSetupTestRule.mock
 Coroutine setup is the same, except for `Dispatchers.setMain(dispatcher)`, which we don't need. 
 
 ```kotlin
-private val dispatcherTestRule = DispatcherTestRule()
+private val dispatcherTestRule = DatabaseDispatcherTestRule()
 ```
 
 Setting the rules:
@@ -182,13 +182,11 @@ mockServerScenarioSetup.setScenario(
 )
 ```
 
-Then we wait for the idling resources, more precisely for the app to navigate us correctly to AuthScreen since we're not logged in:
+Then we wait a bit, more precisely we wait for the app to navigate us correctly to AuthScreen since we're not logged in:
 ```kotlin
-composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+composeTestRule.mainClock.advanceTimeBy(510L)
 ```
 
-> Note: Considering what the docs say this shouldn't be necessarily if the idling resources are setup in Espresso, since the compose test rule is aware of espresso and it waits for idle before every finder. In practice it only works with the line above. Could be a bug somewhere.  
-
 We assert that we are indeed on the correct screen
 ```kotlin
 navigationRobot.assertAuthScreen()
@@ -216,17 +214,20 @@ composeTestRule.mainClock.autoAdvance = true // Let clock auto advance again
 
 Lastly we check the navigation was correct, meaning we should be on the home screen:
 ```kotlin
-composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } // wait for login network call
+composeTestRule.mainClock.awaitIdlingResources() // wait for login network call idling resource
 navigationRobot.assertHomeScreen()
 ```
 
+> `awaitIdlingResources` is an extension function to await all idling resources.
+> Note: Considering what the docs say this shouldn't be necessarily if the idling resources are setup in Espresso, since the compose test rule is aware of espresso and it waits for idle before every finder. In practice it only works with the line above. Could be a bug somewhere.
+
 ### 2. `emptyPasswordShowsProperErrorMessage`
 
 Next up we verify what happens if the user doesn't set their password. We don't need a request in this case.
 
 First we check that we are in the write place:
 ```kotlin
-composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+composeTestRule.mainClock.advanceTimeBy(510L)
 navigationRobot.assertAuthScreen()
 ```
 
@@ -239,7 +240,7 @@ robot.setUsername("banan")
 
 Finally we let coroutines go and verify the error is shown and we have not navigated:
 ```kotlin
-composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+composeTestRule.mainClock.awaitIdlingResources()
 robot.assertErrorIsShown(R.string.password_is_invalid)
     .assertNotLoading()
 navigationRobot.assertAuthScreen()
@@ -251,7 +252,7 @@ This will be really similar as the previous test, so try to do it on your own. T
 
 Still, here is the complete code:
 ```kotlin
-composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+composeTestRule.mainClock.advanceTimeBy(510L)
 navigationRobot.assertAuthScreen()
 
 robot
@@ -259,7 +260,7 @@ robot
     .assertPassword("banan")
     .clickOnLogin()
 
-composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+composeTestRule.mainClock.awaitIdlingResources()
 robot.assertErrorIsShown(R.string.username_is_invalid)
     .assertNotLoading()
 navigationRobot.assertAuthScreen()
@@ -276,7 +277,7 @@ mockServerScenarioSetup.setScenario(
 
 Now input the credentials and fire the event:
 ```kotlin
-composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+composeTestRule.mainClock.advanceTimeBy(510L)
 navigationRobot.assertAuthScreen()
 robot.setUsername("alma")
     .setPassword("banan")
@@ -292,7 +293,7 @@ composeTestRule.mainClock.autoAdvance = true
 
 Now at the end verify the error is shown properly:
 ```kotlin
-composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+composeTestRule.mainClock.awaitIdlingResources()
 robot.assertErrorIsShown(R.string.credentials_invalid)
     .assertNotLoading()
 navigationRobot.assertAuthScreen()
@@ -309,7 +310,7 @@ mockServerScenarioSetup.setScenario(
     AuthScenario.GenericError(username = "alma", password = "banan")
 )
 
-composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+composeTestRule.mainClock.advanceTimeBy(510L)
 navigationRobot.assertAuthScreen()
 robot.setUsername("alma")
     .setPassword("banan")
@@ -322,7 +323,7 @@ composeTestRule.mainClock.advanceTimeByFrame()
 robot.assertLoading()
 composeTestRule.mainClock.autoAdvance = true
 
-composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+composeTestRule.mainClock.awaitIdlingResources()
 robot.assertErrorIsShown(R.string.something_went_wrong)
     .assertNotLoading()
 navigationRobot.assertAuthScreen()
@@ -342,7 +343,7 @@ Then in `setup()`, we need to `setContent` on `stateRestorationTester` instead o
 Now for the actual test, we first setup the content then we trigger restoration by calling `stateRestorationTester.emulateSavedInstanceStateRestore()`, afterwards we can verify that the content is recreated in the correct way:
 
 ```kotlin
-composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() }
+composeTestRule.mainClock.advanceTimeBy(510L)
 navigationRobot.assertAuthScreen()
 robot.setUsername("alma")
     .setPassword("banan")

test-util-android/src/main/java/org/fnives/test/showcase/android/testutil/synchronization/MainDispatcherTestRule.kt

@@ -7,7 +7,7 @@ import kotlinx.coroutines.test.TestDispatcher
 import kotlinx.coroutines.test.UnconfinedTestDispatcher
 import kotlinx.coroutines.test.resetMain
 import kotlinx.coroutines.test.setMain
-import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.anyResourceIdling
+import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.anyResourceNotIdle
 import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.awaitIdlingResources
 import org.junit.rules.TestRule
 import org.junit.runner.Description
@@ -54,7 +54,7 @@ open class MainDispatcherTestRule(private val useStandard: Boolean = true) : Tes
     companion object {
         fun TestDispatcher.advanceUntilIdleWithIdlingResources() {
             scheduler.advanceUntilIdle() // advance until a request is sent
-            while (anyResourceIdling()) { // check if any request is in progress
+            while (anyResourceNotIdle()) { // check if any request is in progress
                 awaitIdlingResources() // complete all requests and other idling resources
                 scheduler.advanceUntilIdle() // run coroutines after request is finished
             }

test-util-android/src/main/java/org/fnives/test/showcase/android/testutil/synchronization/idlingresources/IdlingResourcesHelper.kt

@@ -6,7 +6,7 @@ import org.fnives.test.showcase.android.testutil.synchronization.loopMainThreadF
 import java.util.concurrent.Executors
 
 // workaround, issue with idlingResources is tracked here https://github.com/robolectric/robolectric/issues/4807
-fun anyResourceIdling(): Boolean = !IdlingRegistry.getInstance().resources.all(IdlingResource::isIdleNow)
+fun anyResourceNotIdle(): Boolean = (!IdlingRegistry.getInstance().resources.all(IdlingResource::isIdleNow))
 
 fun awaitIdlingResources() {
     val idlingRegistry = IdlingRegistry.getInstance()
@@ -30,7 +30,7 @@ fun awaitIdlingResources() {
     executor.shutdown()
 }
 
-private fun IdlingResource.awaitUntilIdle() {
+fun IdlingResource.awaitUntilIdle() {
     // using loop because some times, registerIdleTransitionCallback wasn't called
     while (true) {
         if (isIdleNow) return