Add eventually utility

docs/tests.md

@@ -496,6 +496,64 @@ use historical
 to keep track of how frequently flaky tests are failing and to get a better
 understanding of when and why they are failing.
 
+## Retry failing assertions
+
+Use `eventually()` to re-evaluate a body until its assertions stop failing, for
+a test that waits on something outside its control such as a background write or
+an eventually-consistent read.
+
+In order to use it, you must first build an instance of `EventuallyOptions` with
+`EventuallyOptions(retries, sleep)` (both parameters must be positive) and modify
+using `withMaxRetries`, `withSleep`, `withFilters` (or `addFilters`). You can
+also use `EventuallyOptions.disabled`, which runs the body without retrying.
+
+> On Scala.js a body that is not async cannot sleep between attempts, so its
+> retries run back to back.
+
+The filters are predicates taking a `Throwable` and returning a Boolean, to
+designate which exceptions we will retry on. By default, only failed munit
+assertions (derived from `munit.FailExceptionLike`) are retried. `addFilters`
+adds filters to the current set, while `withFilters` replaces it.
+
+`eventually()` needs an `EventuallyOptions` in implicit scope. (As is usual
+with implicits, if you declare one for the whole suite and want to override it
+in a narrower scope, you must give the inner one the same name.) You can also
+call `eventually()` directly on an `EventuallyOptions` when a single call
+needs its own settings.
+
+```scala mdoc
+import munit.EventuallyOptions
+
+import scala.concurrent.duration._
+
+class EventuallySuite extends munit.FunSuite {
+  private implicit val options: EventuallyOptions =
+    EventuallyOptions(40, 100.millis)
+
+  // a query against a table another process writes to
+  def rowCount: Int = ???
+
+  test("rows arrive") {
+    eventually(assertEquals(rowCount, 3))
+  }
+
+  test("this table takes longer to fill") {
+    EventuallyOptions(60, 1.second).eventually(assertEquals(rowCount, 3))
+  }
+
+  test("the connection also comes up late") {
+    options.addFilters(_.isInstanceOf[java.io.IOException])
+      .eventually(assertEquals(rowCount, 3))
+  }
+
+  test("this whole test waits longer") {
+    implicit val options: EventuallyOptions = this.options.withMaxRetries(120)
+    eventually(assert(rowCount > 0))
+    eventually(assertEquals(rowCount, 3))
+  }
+}
+```
+
 ## Run logic before and after tests
 
 See the [fixtures guide](fixtures.html) for instructions for running custom

munit/js/src/main/scala/munit/internal/PlatformCompat.scala

@@ -6,7 +6,7 @@ import sbt.testing.{EventHandler, Logger, Task, TaskDef}
 
 import java.util.concurrent.TimeoutException
 
-import scala.concurrent.duration.Duration
+import scala.concurrent.duration.{Duration, FiniteDuration}
 import scala.concurrent.{Await, Awaitable, ExecutionContext, Future, Promise}
 import scala.scalajs.js.timers
 import scala.scalajs.reflect.Reflect
@@ -20,6 +20,9 @@ object PlatformCompat {
   def awaitResult[T](awaitable: Awaitable[T]): T = Await
     .result(awaitable, Duration.Inf)
 
+  // Scala.js is single threaded, so we can't block.
+  def sleep(duration: FiniteDuration): Unit = ()
+
   def executeAsync(
       task: Task,
       eventHandler: EventHandler,

munit/jvm/src/main/scala/munit/internal/PlatformCompat.scala

@@ -7,7 +7,7 @@ import java.util.concurrent.{
   Executors, ThreadFactory, TimeUnit, TimeoutException,
 }
 
-import scala.concurrent.duration.Duration
+import scala.concurrent.duration.{Duration, FiniteDuration}
 import scala.concurrent.{Await, Awaitable, ExecutionContext, Future, Promise}
 import scala.util.control.NonFatal
 
@@ -29,6 +29,8 @@ object PlatformCompat {
   def awaitResult[T](awaitable: Awaitable[T]): T = Await
     .result(awaitable, Duration.Inf)
 
+  def sleep(duration: FiniteDuration): Unit = Thread.sleep(duration.toMillis)
+
   def executeAsync(
       task: Task,
       eventHandler: EventHandler,

munit/native/src/main/scala/munit/internal/PlatformCompat.scala

@@ -9,7 +9,7 @@ import java.util.concurrent.{
   Executors, ThreadFactory, TimeUnit, TimeoutException,
 }
 
-import scala.concurrent.duration.Duration
+import scala.concurrent.duration.{Duration, FiniteDuration}
 import scala.concurrent.{Await, Awaitable, ExecutionContext, Future, Promise}
 import scala.scalanative.meta.LinktimeInfo.isMultithreadingEnabled
 import scala.scalanative.reflect.Reflect
@@ -39,6 +39,8 @@ object PlatformCompat {
     Await.result(awaitable, Duration.Inf)
   }
 
+  def sleep(duration: FiniteDuration): Unit = Thread.sleep(duration.toMillis)
+
   def executeAsync(
       task: Task,
       eventHandler: EventHandler,

munit/shared/src/main/scala/munit/EventuallyOptions.scala

@@ -0,0 +1,106 @@
+package munit
+
+import munit.internal.{PlatformCompat, console}
+
+import scala.concurrent._
+import scala.concurrent.duration._
+import scala.util._
+
+final class EventuallyOptions private (
+    val maxRetries: Int,
+    val sleep: FiniteDuration,
+    private val filters: Seq[Throwable => Boolean] =
+      Seq(EventuallyOptions.isAssertion),
+    // add fields above with defaults, to `privateCopy`, then a `withX` method
+) {
+
+  /**
+   * Whether `eventually` retries after the given exception.
+   *
+   * A `Future` boxes an `AssertionError` in an `ExecutionException`, so the
+   * filter is given the root cause and matches alike on either path.
+   */
+  private def isRetriable(throwable: Throwable): Boolean = {
+    val ex = Exceptions.rootCause(throwable)
+    filters.exists(_(ex))
+  }
+
+  def withMaxRetries(newValue: Int): EventuallyOptions =
+    privateCopy(maxRetries = EventuallyOptions.requirePositive(newValue))
+  def withSleep(newValue: FiniteDuration): EventuallyOptions =
+    privateCopy(sleep = EventuallyOptions.requirePositive(newValue))
+  def withFilters(newValue: Throwable => Boolean*): EventuallyOptions = {
+    require(newValue.nonEmpty, "filters must be non-empty")
+    privateCopy(filters = newValue)
+  }
+  def addFilters(newValue: Throwable => Boolean*): EventuallyOptions =
+    privateCopy(filters = this.filters ++ newValue)
+
+  private[this] def privateCopy(
+      maxRetries: Int = this.maxRetries,
+      sleep: FiniteDuration = this.sleep,
+      filters: Seq[Throwable => Boolean] = this.filters,
+  ): EventuallyOptions = new EventuallyOptions(maxRetries, sleep, filters)
+
+  /**
+   * Evaluates the effectful body until it succeeds or max retries are exhausted.
+   */
+  def eventually[A](body: => A)(implicit transform: EventuallyTransform[A]): A =
+    transform(body, this)
+
+  private[munit] def retryAsync[A](
+      body: => Future[A]
+  )(implicit ctx: SuiteContext): Future[A] = {
+    implicit val ec: ExecutionContext = ctx.ec
+    def attempt(remaining: Int): Future[A] = Future.unit
+      .flatMap(_ => console.StackTraces.dropOutside(body)).transformWith {
+        case Failure(ex) if remaining > 0 && isRetriable(ex) =>
+          val promise = Promise[Unit]()
+          PlatformCompat.setTimeout(sleep.toMillis.toInt) {
+            promise.trySuccess(())
+            ()
+          }
+          promise.future.flatMap(_ => attempt(remaining - 1))
+        case x => Future.fromTry(x)
+      }
+
+    attempt(maxRetries)
+  }
+
+  private[munit] def retry[A](body: => A): A = {
+    // a loop, not recursion: retries must not deepen the reported stack trace
+    var remaining = maxRetries
+    while (remaining > 0)
+      try return console.StackTraces.dropOutside(body)
+      catch {
+        case control.NonFatal(ex) if isRetriable(ex) =>
+          PlatformCompat.sleep(sleep)
+          remaining -= 1
+      }
+    console.StackTraces.dropOutside(body)
+  }
+
+}
+
+object EventuallyOptions {
+
+  def apply(maxRetries: Int, sleep: FiniteDuration) =
+    new EventuallyOptions(requirePositive(maxRetries), requirePositive(sleep))
+
+  private val isAssertion =
+    (ex: Throwable) => ex.isInstanceOf[FailExceptionLike[_]]
+
+  /** Retries nothing, so `eventually` evaluates its body exactly once. */
+  val disabled: EventuallyOptions = new EventuallyOptions(0, Duration.Zero)
+
+  private def requirePositive(maxRetries: Int): Int = {
+    require(maxRetries > 0, s"maxRetries must be positive: $maxRetries")
+    maxRetries
+  }
+
+  private def requirePositive(sleep: FiniteDuration): FiniteDuration = {
+    require(sleep > Duration.Zero, s"sleep must be positive: $sleep")
+    sleep
+  }
+
+}

munit/shared/src/main/scala/munit/EventuallyTransform.scala

@@ -0,0 +1,33 @@
+package munit
+
+import scala.concurrent.Future
+
+/**
+ * Picks how `eventually` retries a body of type `A`.
+ *
+ * Dispatch is by type rather than by overload: a `Future` body must retry
+ * without blocking, anything else retries in place.
+ */
+trait EventuallyTransform[A] {
+  def apply(body: => A, options: EventuallyOptions): A
+}
+
+object EventuallyTransform extends EventuallyTransformLowPriority {
+
+  implicit def munitFutureTransform[A](implicit
+      ctx: SuiteContext
+  ): EventuallyTransform[Future[A]] = new EventuallyTransform[Future[A]] {
+    def apply(body: => Future[A], options: EventuallyOptions): Future[A] =
+      options.retryAsync(body)
+  }
+
+}
+
+trait EventuallyTransformLowPriority {
+
+  implicit def munitValueTransform[A]: EventuallyTransform[A] =
+    new EventuallyTransform[A] {
+      def apply(body: => A, options: EventuallyOptions): A = options.retry(body)
+    }
+
+}

munit/shared/src/main/scala/munit/EventuallyTransforms.scala

@@ -0,0 +1,12 @@
+package munit
+
+trait EventuallyTransforms {
+  this: BaseFunSuite =>
+
+  /** Evaluates the effectful body until it succeeds or max retries are exhausted. */
+  def eventually[A](
+      body: => A
+  )(implicit options: EventuallyOptions, transform: EventuallyTransform[A]): A =
+    options.eventually(body)
+
+}

munit/shared/src/main/scala/munit/FunSuite.scala

@@ -5,8 +5,8 @@ import munit.internal.PlatformCompat
 import java.util.concurrent.TimeUnit
 
 import scala.collection.mutable
-import scala.concurrent.Future
 import scala.concurrent.duration.{Duration, FiniteDuration}
+import scala.concurrent.{ExecutionContext, Future}
 import scala.util.control.NonFatal
 
 abstract class FunSuite extends BaseFunSuite
@@ -18,7 +18,8 @@ trait BaseFunSuite
     with TestOptionsConversions
     with TestTransforms
     with SuiteTransforms
-    with ValueTransforms {
+    with ValueTransforms
+    with EventuallyTransforms {
   self =>
 
   final val munitTestsBuffer: mutable.ListBuffer[Test] = mutable.ListBuffer
@@ -42,4 +43,9 @@ trait BaseFunSuite
   private final def waitForCompletion[T](f: () => Future[T]) = PlatformCompat
     .waitAtMost(f, munitTimeout, munitExecutionContext)
 
+  /** Wraps non-implicit fields as implicit */
+  implicit def implicitContext: SuiteContext = new SuiteContext {
+    override def ec: ExecutionContext = munitExecutionContext
+  }
+
 }

munit/shared/src/main/scala/munit/SuiteContext.scala

@@ -0,0 +1,9 @@
+package munit
+
+import scala.concurrent.ExecutionContext
+
+trait SuiteContext {
+
+  def ec: ExecutionContext
+
+}

tests/shared/src/main/scala/munit/EventuallyStackTraceFrameworkSuite.scala

@@ -0,0 +1,37 @@
+package munit
+
+import scala.concurrent.duration._
+
+class EventuallyStackTraceFrameworkSuite extends FunSuite {
+  private implicit val options: EventuallyOptions = EventuallyOptions(4, 1.milli)
+  test("fail")(eventually(assertEquals(1, 2)))
+}
+
+// Retrying must stay out of the reported stack trace: these are the frames a
+// plain failing assertion produces, with no trace of the retry loop.
+object EventuallyStackTraceFrameworkSuite
+    extends FrameworkTest(
+      classOf[EventuallyStackTraceFrameworkSuite],
+      """|at munit.FunSuite:assertEquals
+         |  at munit.EventuallyStackTraceFrameworkSuite:$anonfun$new$2
+         |  at scala.runtime.java8.JFunction0$mcV$sp:apply
+         |==> failure munit.EventuallyStackTraceFrameworkSuite.fail - tests/shared/src/main/scala/munit/EventuallyStackTraceFrameworkSuite.scala:7
+         |6:  private implicit val options: EventuallyOptions = EventuallyOptions(4, 1.milli)
+         |7:  test("fail")(eventually(assertEquals(1, 2)))
+         |8:}
+         |values are not the same
+         |=> Obtained
+         |1
+         |=> Diff (- expected, + obtained)
+         |-2
+         |+1
+         |""".stripMargin,
+      tags = Set(OnlyJVM),
+      onEvent = { event =>
+        if (event.throwable().isDefined()) {
+          val s = event.throwable().get().getStackTrace()
+          s.map(e => s"  at ${e.getClassName()}:${e.getMethodName()}")
+            .mkString("", "\n", "\n")
+        } else ""
+      },
+    )