From a2d018efbfcf38d9280e460aa8cc77c684b2a708 Mon Sep 17 00:00:00 2001 From: Gergely Hegedus Date: Wed, 13 Jul 2022 17:20:59 +0300 Subject: [PATCH] Issue#97 Attempt to fix flakiness in Compose The flakiness is caused by timeout when waiting for idling resources. To circumvent this, we will always await the idling resources, and at the start we will only wait for compose-time, to navigate away from Splash. --- .../ui/AuthComposeInstrumentedTest.kt | 62 +++++++++++++++---- .../idling/DatabaseDispatcherTestRule.kt | 4 +- codekata/compose.instructionset.md | 31 +++++----- .../synchronization/MainDispatcherTestRule.kt | 4 +- .../idlingresources/IdlingResourcesHelper.kt | 4 +- 5 files changed, 72 insertions(+), 33 deletions(-) diff --git a/app/src/androidTest/java/org/fnives/test/showcase/ui/AuthComposeInstrumentedTest.kt b/app/src/androidTest/java/org/fnives/test/showcase/ui/AuthComposeInstrumentedTest.kt index f2666d7..7b2ad93 100644 --- a/app/src/androidTest/java/org/fnives/test/showcase/ui/AuthComposeInstrumentedTest.kt +++ b/app/src/androidTest/java/org/fnives/test/showcase/ui/AuthComposeInstrumentedTest.kt @@ -1,11 +1,16 @@ package org.fnives.test.showcase.ui +import androidx.compose.ui.test.MainTestClock import androidx.compose.ui.test.junit4.StateRestorationTester import androidx.compose.ui.test.junit4.createComposeRule +import androidx.test.espresso.IdlingRegistry +import androidx.test.espresso.IdlingResource import androidx.test.ext.junit.runners.AndroidJUnit4 import org.fnives.test.showcase.R import org.fnives.test.showcase.android.testutil.screenshot.ScreenshotRule -import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.anyResourceIdling +import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.anyResourceNotIdle +import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.awaitUntilIdle +import org.fnives.test.showcase.android.testutil.synchronization.loopMainThreadFor import org.fnives.test.showcase.compose.screen.AppNavigation import org.fnives.test.showcase.core.integration.fake.FakeUserDataLocalStorage import org.fnives.test.showcase.core.login.IsUserLoggedInUseCase @@ -18,6 +23,7 @@ import org.junit.Test import org.junit.rules.RuleChain import org.junit.runner.RunWith import org.koin.test.KoinTest +import java.util.concurrent.Executors @RunWith(AndroidJUnit4::class) class AuthComposeInstrumentedTest : KoinTest { @@ -53,8 +59,8 @@ class AuthComposeInstrumentedTest : KoinTest { mockServerScenarioSetup.setScenario( AuthScenario.Success(password = "alma", username = "banan") ) + composeTestRule.mainClock.advanceTimeBy(SPLASH_DELAY) - composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } navigationRobot.assertAuthScreen() robot.setPassword("alma") .setUsername("banan") @@ -67,21 +73,21 @@ class AuthComposeInstrumentedTest : KoinTest { robot.assertLoading() composeTestRule.mainClock.autoAdvance = true - composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } + composeTestRule.mainClock.awaitIdlingResources() navigationRobot.assertHomeScreen() } /** GIVEN empty password and username WHEN signIn THEN error password is shown */ @Test fun emptyPasswordShowsProperErrorMessage() { - composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } + composeTestRule.mainClock.advanceTimeBy(SPLASH_DELAY) navigationRobot.assertAuthScreen() robot.setUsername("banan") .assertUsername("banan") .clickOnLogin() - composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } + composeTestRule.mainClock.awaitIdlingResources() robot.assertErrorIsShown(R.string.password_is_invalid) .assertNotLoading() navigationRobot.assertAuthScreen() @@ -90,7 +96,7 @@ class AuthComposeInstrumentedTest : KoinTest { /** GIVEN password and empty username WHEN signIn THEN error username is shown */ @Test fun emptyUserNameShowsProperErrorMessage() { - composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } + composeTestRule.mainClock.advanceTimeBy(SPLASH_DELAY) navigationRobot.assertAuthScreen() robot @@ -98,7 +104,7 @@ class AuthComposeInstrumentedTest : KoinTest { .assertPassword("banan") .clickOnLogin() - composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } + composeTestRule.mainClock.awaitIdlingResources() robot.assertErrorIsShown(R.string.username_is_invalid) .assertNotLoading() navigationRobot.assertAuthScreen() @@ -111,7 +117,7 @@ class AuthComposeInstrumentedTest : KoinTest { AuthScenario.InvalidCredentials(password = "alma", username = "banan") ) - composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } + composeTestRule.mainClock.advanceTimeBy(SPLASH_DELAY) navigationRobot.assertAuthScreen() robot.setUsername("alma") .setPassword("banan") @@ -124,7 +130,7 @@ class AuthComposeInstrumentedTest : KoinTest { robot.assertLoading() composeTestRule.mainClock.autoAdvance = true - composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } + composeTestRule.mainClock.awaitIdlingResources() robot.assertErrorIsShown(R.string.credentials_invalid) .assertNotLoading() navigationRobot.assertAuthScreen() @@ -137,7 +143,7 @@ class AuthComposeInstrumentedTest : KoinTest { AuthScenario.GenericError(username = "alma", password = "banan") ) - composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } + composeTestRule.mainClock.advanceTimeBy(SPLASH_DELAY) navigationRobot.assertAuthScreen() robot.setUsername("alma") .setPassword("banan") @@ -150,7 +156,7 @@ class AuthComposeInstrumentedTest : KoinTest { robot.assertLoading() composeTestRule.mainClock.autoAdvance = true - composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } + composeTestRule.mainClock.awaitIdlingResources() robot.assertErrorIsShown(R.string.something_went_wrong) .assertNotLoading() navigationRobot.assertAuthScreen() @@ -159,7 +165,7 @@ class AuthComposeInstrumentedTest : KoinTest { /** GIVEN username and password WHEN restoring THEN username and password fields contain the same text */ @Test fun restoringContentShowPreviousCredentials() { - composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } + composeTestRule.mainClock.advanceTimeBy(SPLASH_DELAY) navigationRobot.assertAuthScreen() robot.setUsername("alma") .setPassword("banan") @@ -172,4 +178,36 @@ class AuthComposeInstrumentedTest : KoinTest { robot.assertUsername("alma") .assertPassword("banan") } + + companion object { + private const val SPLASH_DELAY = 600L + + // workaround, issue with idlingResources is tracked here https://github.com/robolectric/robolectric/issues/4807 + /** + * Await the idling resource on a different thread while looping main. + */ + fun MainTestClock.awaitIdlingResources() { + val idlingRegistry = IdlingRegistry.getInstance() + if (!anyResourceNotIdle()) return + + val executor = Executors.newSingleThreadExecutor() + var isIdle = false + executor.submit { + do { + idlingRegistry.resources + .filterNot(IdlingResource::isIdleNow) + .forEach { idlingResource -> + idlingResource.awaitUntilIdle() + } + } while (!idlingRegistry.resources.all(IdlingResource::isIdleNow)) + isIdle = true + } + while (!isIdle) { + loopMainThreadFor(200L) + } + executor.shutdown() + + advanceTimeByFrame() + } + } } diff --git a/app/src/sharedTest/java/org/fnives/test/showcase/testutils/idling/DatabaseDispatcherTestRule.kt b/app/src/sharedTest/java/org/fnives/test/showcase/testutils/idling/DatabaseDispatcherTestRule.kt index 4708d8b..59ea741 100644 --- a/app/src/sharedTest/java/org/fnives/test/showcase/testutils/idling/DatabaseDispatcherTestRule.kt +++ b/app/src/sharedTest/java/org/fnives/test/showcase/testutils/idling/DatabaseDispatcherTestRule.kt @@ -3,7 +3,7 @@ package org.fnives.test.showcase.testutils.idling import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.test.StandardTestDispatcher import kotlinx.coroutines.test.TestDispatcher -import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.anyResourceIdling +import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.anyResourceNotIdle import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.awaitIdlingResources import org.fnives.test.showcase.android.testutil.synchronization.runOnUIAwaitOnCurrent import org.fnives.test.showcase.testutils.storage.TestDatabaseInitialization @@ -42,7 +42,7 @@ class DatabaseDispatcherTestRule : TestRule { companion object { fun TestDispatcher.advanceUntilIdleWithIdlingResources() { scheduler.advanceUntilIdle() // advance until a request is sent - while (anyResourceIdling()) { // check if any request is in progress + while (anyResourceNotIdle()) { // check if any request is in progress awaitIdlingResources() // complete all requests and other idling resources scheduler.advanceUntilIdle() // run coroutines after request is finished } diff --git a/codekata/compose.instructionset.md b/codekata/compose.instructionset.md index 2de8d66..ce50776 100644 --- a/codekata/compose.instructionset.md +++ b/codekata/compose.instructionset.md @@ -159,7 +159,7 @@ private val mockServerScenarioSetup get() = mockServerScenarioSetupTestRule.mock Coroutine setup is the same, except for `Dispatchers.setMain(dispatcher)`, which we don't need. ```kotlin -private val dispatcherTestRule = DispatcherTestRule() +private val dispatcherTestRule = DatabaseDispatcherTestRule() ``` Setting the rules: @@ -182,13 +182,11 @@ mockServerScenarioSetup.setScenario( ) ``` -Then we wait for the idling resources, more precisely for the app to navigate us correctly to AuthScreen since we're not logged in: +Then we wait a bit, more precisely we wait for the app to navigate us correctly to AuthScreen since we're not logged in: ```kotlin -composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } +composeTestRule.mainClock.advanceTimeBy(510L) ``` -> Note: Considering what the docs say this shouldn't be necessarily if the idling resources are setup in Espresso, since the compose test rule is aware of espresso and it waits for idle before every finder. In practice it only works with the line above. Could be a bug somewhere. - We assert that we are indeed on the correct screen ```kotlin navigationRobot.assertAuthScreen() @@ -216,17 +214,20 @@ composeTestRule.mainClock.autoAdvance = true // Let clock auto advance again Lastly we check the navigation was correct, meaning we should be on the home screen: ```kotlin -composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } // wait for login network call +composeTestRule.mainClock.awaitIdlingResources() // wait for login network call idling resource navigationRobot.assertHomeScreen() ``` +> `awaitIdlingResources` is an extension function to await all idling resources. +> Note: Considering what the docs say this shouldn't be necessarily if the idling resources are setup in Espresso, since the compose test rule is aware of espresso and it waits for idle before every finder. In practice it only works with the line above. Could be a bug somewhere. + ### 2. `emptyPasswordShowsProperErrorMessage` Next up we verify what happens if the user doesn't set their password. We don't need a request in this case. First we check that we are in the write place: ```kotlin -composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } +composeTestRule.mainClock.advanceTimeBy(510L) navigationRobot.assertAuthScreen() ``` @@ -239,7 +240,7 @@ robot.setUsername("banan") Finally we let coroutines go and verify the error is shown and we have not navigated: ```kotlin -composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } +composeTestRule.mainClock.awaitIdlingResources() robot.assertErrorIsShown(R.string.password_is_invalid) .assertNotLoading() navigationRobot.assertAuthScreen() @@ -251,7 +252,7 @@ This will be really similar as the previous test, so try to do it on your own. T Still, here is the complete code: ```kotlin -composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } +composeTestRule.mainClock.advanceTimeBy(510L) navigationRobot.assertAuthScreen() robot @@ -259,7 +260,7 @@ robot .assertPassword("banan") .clickOnLogin() -composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } +composeTestRule.mainClock.awaitIdlingResources() robot.assertErrorIsShown(R.string.username_is_invalid) .assertNotLoading() navigationRobot.assertAuthScreen() @@ -276,7 +277,7 @@ mockServerScenarioSetup.setScenario( Now input the credentials and fire the event: ```kotlin -composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } +composeTestRule.mainClock.advanceTimeBy(510L) navigationRobot.assertAuthScreen() robot.setUsername("alma") .setPassword("banan") @@ -292,7 +293,7 @@ composeTestRule.mainClock.autoAdvance = true Now at the end verify the error is shown properly: ```kotlin -composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } +composeTestRule.mainClock.awaitIdlingResources() robot.assertErrorIsShown(R.string.credentials_invalid) .assertNotLoading() navigationRobot.assertAuthScreen() @@ -309,7 +310,7 @@ mockServerScenarioSetup.setScenario( AuthScenario.GenericError(username = "alma", password = "banan") ) -composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } +composeTestRule.mainClock.advanceTimeBy(510L) navigationRobot.assertAuthScreen() robot.setUsername("alma") .setPassword("banan") @@ -322,7 +323,7 @@ composeTestRule.mainClock.advanceTimeByFrame() robot.assertLoading() composeTestRule.mainClock.autoAdvance = true -composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } +composeTestRule.mainClock.awaitIdlingResources() robot.assertErrorIsShown(R.string.something_went_wrong) .assertNotLoading() navigationRobot.assertAuthScreen() @@ -342,7 +343,7 @@ Then in `setup()`, we need to `setContent` on `stateRestorationTester` instead o Now for the actual test, we first setup the content then we trigger restoration by calling `stateRestorationTester.emulateSavedInstanceStateRestore()`, afterwards we can verify that the content is recreated in the correct way: ```kotlin -composeTestRule.mainClock.advanceTimeUntil { anyResourceIdling() } +composeTestRule.mainClock.advanceTimeBy(510L) navigationRobot.assertAuthScreen() robot.setUsername("alma") .setPassword("banan") diff --git a/test-util-android/src/main/java/org/fnives/test/showcase/android/testutil/synchronization/MainDispatcherTestRule.kt b/test-util-android/src/main/java/org/fnives/test/showcase/android/testutil/synchronization/MainDispatcherTestRule.kt index 8e2676d..69ba21a 100644 --- a/test-util-android/src/main/java/org/fnives/test/showcase/android/testutil/synchronization/MainDispatcherTestRule.kt +++ b/test-util-android/src/main/java/org/fnives/test/showcase/android/testutil/synchronization/MainDispatcherTestRule.kt @@ -7,7 +7,7 @@ import kotlinx.coroutines.test.TestDispatcher import kotlinx.coroutines.test.UnconfinedTestDispatcher import kotlinx.coroutines.test.resetMain import kotlinx.coroutines.test.setMain -import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.anyResourceIdling +import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.anyResourceNotIdle import org.fnives.test.showcase.android.testutil.synchronization.idlingresources.awaitIdlingResources import org.junit.rules.TestRule import org.junit.runner.Description @@ -54,7 +54,7 @@ open class MainDispatcherTestRule(private val useStandard: Boolean = true) : Tes companion object { fun TestDispatcher.advanceUntilIdleWithIdlingResources() { scheduler.advanceUntilIdle() // advance until a request is sent - while (anyResourceIdling()) { // check if any request is in progress + while (anyResourceNotIdle()) { // check if any request is in progress awaitIdlingResources() // complete all requests and other idling resources scheduler.advanceUntilIdle() // run coroutines after request is finished } diff --git a/test-util-android/src/main/java/org/fnives/test/showcase/android/testutil/synchronization/idlingresources/IdlingResourcesHelper.kt b/test-util-android/src/main/java/org/fnives/test/showcase/android/testutil/synchronization/idlingresources/IdlingResourcesHelper.kt index e8288f7..7246c54 100644 --- a/test-util-android/src/main/java/org/fnives/test/showcase/android/testutil/synchronization/idlingresources/IdlingResourcesHelper.kt +++ b/test-util-android/src/main/java/org/fnives/test/showcase/android/testutil/synchronization/idlingresources/IdlingResourcesHelper.kt @@ -6,7 +6,7 @@ import org.fnives.test.showcase.android.testutil.synchronization.loopMainThreadF import java.util.concurrent.Executors // workaround, issue with idlingResources is tracked here https://github.com/robolectric/robolectric/issues/4807 -fun anyResourceIdling(): Boolean = !IdlingRegistry.getInstance().resources.all(IdlingResource::isIdleNow) +fun anyResourceNotIdle(): Boolean = (!IdlingRegistry.getInstance().resources.all(IdlingResource::isIdleNow)) fun awaitIdlingResources() { val idlingRegistry = IdlingRegistry.getInstance() @@ -30,7 +30,7 @@ fun awaitIdlingResources() { executor.shutdown() } -private fun IdlingResource.awaitUntilIdle() { +fun IdlingResource.awaitUntilIdle() { // using loop because some times, registerIdleTransitionCallback wasn't called while (true) { if (isIdleNow) return