[WOR-1297] Continue workspace deletion if all apps/runtimes in are ERROR status

core/src/main/scala/org/broadinstitute/dsde/rawls/monitor/workspace/runners/deletion/actions/LeonardoDeletionActions.scala

@@ -8,7 +8,12 @@ import org.broadinstitute.dsde.rawls.model.{RawlsRequestContext, Workspace}
 import org.broadinstitute.dsde.rawls.monitor.workspace.runners.deletion.actions.DeletionAction.when500OrProcessingException
 import org.broadinstitute.dsde.rawls.util.Retry
 import org.broadinstitute.dsde.workbench.client.leonardo.ApiException
-import org.broadinstitute.dsde.workbench.client.leonardo.model.{ListAppResponse, ListRuntimeResponse}
+import org.broadinstitute.dsde.workbench.client.leonardo.model.{
+  AppStatus,
+  ClusterStatus,
+  ListAppResponse,
+  ListRuntimeResponse
+}
 
 import java.util.UUID
 import scala.concurrent.{blocking, ExecutionContext, Future}
@@ -45,34 +50,50 @@ class LeonardoResourceDeletionAction(leonardoDAO: LeonardoDAO)(implicit
     ec: ExecutionContext
   ): Future[Boolean] = {
     logger.info(s"Polling runtime deletion [workspaceId=${workspace.workspaceId}]")
-    pollOperation[ListRuntimeResponse](workspace, ctx, listAzureRuntimes)
+    pollOperation[ListRuntimeResponse](workspace, ctx, listNonErroredAzureRuntimes)
   }
 
   def pollAppDeletion(workspace: Workspace, ctx: RawlsRequestContext)(implicit
     ec: ExecutionContext
   ): Future[Boolean] = {
     logger.info(s"Polling app deletion [workspaceId=${workspace.workspaceId}]")
-    pollOperation[ListAppResponse](workspace, ctx, listApps)
+    pollOperation[ListAppResponse](workspace, ctx, listNonErroredApps)
   }
 
-  def listApps(workspace: Workspace, ctx: RawlsRequestContext)(implicit
+  def listNonErroredApps(workspace: Workspace, ctx: RawlsRequestContext)(implicit
     ec: ExecutionContext
   ): Future[Seq[ListAppResponse]] =
     retry(when500OrProcessingException) { () =>
       Future {
         blocking {
-          leonardoDAO.listApps(ctx.userInfo.accessToken.token, workspace.workspaceIdAsUUID)
+          val allApps = leonardoDAO.listApps(ctx.userInfo.accessToken.token, workspace.workspaceIdAsUUID)
+          val nonErroredApps = allApps.filter(_.getStatus != AppStatus.ERROR)
+          val erroredAppCount = allApps.size - nonErroredApps.size
+          if (erroredAppCount > 0) {
+            logger.info(
+              s"Filtering out ${erroredAppCount} errored apps for [workspaceId=${workspace.workspaceIdAsUUID}]"
+            )
+          }
+          nonErroredApps
         }
       }
     }
 
-  def listAzureRuntimes(workspace: Workspace, ctx: RawlsRequestContext)(implicit
+  def listNonErroredAzureRuntimes(workspace: Workspace, ctx: RawlsRequestContext)(implicit
     ec: ExecutionContext
   ): Future[Seq[ListRuntimeResponse]] =
     retry(when500OrProcessingException) { () =>
       Future {
         blocking {
-          leonardoDAO.listAzureRuntimes(ctx.userInfo.accessToken.token, workspace.workspaceIdAsUUID)
+          val allRuntimes = leonardoDAO.listAzureRuntimes(ctx.userInfo.accessToken.token, workspace.workspaceIdAsUUID)
+          val nonErroredRuntimes = allRuntimes.filter(_.getStatus != ClusterStatus.ERROR)
+          val erroredRuntimeCount = allRuntimes.size - nonErroredRuntimes.size
+          if (erroredRuntimeCount > 0) {
+            logger.info(
+              s"Filtering out ${erroredRuntimeCount} errored runtimes for [workspaceId=${workspace.workspaceIdAsUUID}]"
+            )
+          }
+          nonErroredRuntimes
         }
       }
     }

core/src/test/scala/org/broadinstitute/dsde/rawls/monitor/workspace/runners/deletion/actions/LeonardoResourceDeletionActionSpec.scala

@@ -13,7 +13,12 @@ import org.broadinstitute.dsde.rawls.model.{
   Workspace
 }
 import org.broadinstitute.dsde.workbench.client.leonardo.ApiException
-import org.broadinstitute.dsde.workbench.client.leonardo.model.{ListAppResponse, ListRuntimeResponse}
+import org.broadinstitute.dsde.workbench.client.leonardo.model.{
+  AppStatus,
+  ClusterStatus,
+  ListAppResponse,
+  ListRuntimeResponse
+}
 import org.joda.time.DateTime
 import org.mockito.ArgumentMatchers.{any, anyString}
 import org.mockito.Mockito.{times, verify, when, RETURNS_SMART_NULLS}
@@ -24,8 +29,7 @@ import org.scalatest.matchers.should.Matchers
 import org.scalatestplus.mockito.MockitoSugar
 
 import java.util.UUID
-import java.util.concurrent.TimeUnit
-import scala.concurrent.duration.{Duration, FiniteDuration}
+import scala.concurrent.duration.Duration
 import scala.concurrent.{Await, ExecutionContext}
 
 class LeonardoResourceDeletionActionSpec extends AnyFlatSpec with MockitoSugar with Matchers with ScalaFutures {
@@ -113,9 +117,14 @@ class LeonardoResourceDeletionActionSpec extends AnyFlatSpec with MockitoSugar w
 
   behavior of "pollAppDeletion"
 
-  it should "poll and return false when apps have not finished deleting" in {
+  it should "poll and return false when apps have not finished deleting and are not all in the error state" in {
+    val deletingAppResponse = new ListAppResponse()
+    deletingAppResponse.setStatus(AppStatus.DELETING)
+    val erroredAppResponse = new ListAppResponse()
+    erroredAppResponse.setStatus(AppStatus.ERROR)
     val leoDAO: MockLeonardoDAO = Mockito.spy(new MockLeonardoDAO() {
-      override def listApps(token: String, workspaceId: UUID): Seq[ListAppResponse] = Seq(new ListAppResponse())
+      override def listApps(token: String, workspaceId: UUID): Seq[ListAppResponse] =
+        Seq(deletingAppResponse, erroredAppResponse)
     })
 
     val action = new LeonardoResourceDeletionAction(leoDAO)
@@ -124,6 +133,20 @@ class LeonardoResourceDeletionActionSpec extends AnyFlatSpec with MockitoSugar w
     verify(leoDAO).listApps(anyString(), any[UUID])
   }
 
+  it should "poll and return true when all apps are in the error state" in {
+    val erroredAppResponse = new ListAppResponse()
+    erroredAppResponse.setStatus(AppStatus.ERROR)
+    val leoDAO: MockLeonardoDAO = Mockito.spy(new MockLeonardoDAO() {
+      override def listApps(token: String, workspaceId: UUID): Seq[ListAppResponse] =
+        Seq(erroredAppResponse, erroredAppResponse)
+    })
+
+    val action = new LeonardoResourceDeletionAction(leoDAO)
+
+    Await.result(action.pollAppDeletion(azureWorkspace, ctx), Duration.Inf) shouldBe true
+    verify(leoDAO).listApps(anyString(), any[UUID])
+  }
+
   it should "poll and return true when apps have finished deleting" in {
     val leoDAO: MockLeonardoDAO = Mockito.spy(new MockLeonardoDAO() {
       override def listApps(token: String, workspaceId: UUID): Seq[ListAppResponse] = Seq.empty
@@ -197,4 +220,34 @@ class LeonardoResourceDeletionActionSpec extends AnyFlatSpec with MockitoSugar w
     }
   }
 
+  it should "poll and return false when runtimes have not finished deleting and are not all in the error state" in {
+    val deletingRuntimeResponse = new ListRuntimeResponse()
+    deletingRuntimeResponse.setStatus(ClusterStatus.DELETING)
+    val erroredRuntimeResponse = new ListRuntimeResponse()
+    erroredRuntimeResponse.setStatus(ClusterStatus.ERROR)
+    val leoDAO: MockLeonardoDAO = Mockito.spy(new MockLeonardoDAO() {
+      override def listAzureRuntimes(token: String, workspaceId: UUID): Seq[ListRuntimeResponse] =
+        Seq(deletingRuntimeResponse, erroredRuntimeResponse)
+    })
+
+    val action = new LeonardoResourceDeletionAction(leoDAO)
+
+    Await.result(action.pollRuntimeDeletion(azureWorkspace, ctx), Duration.Inf) shouldBe false
+    verify(leoDAO).listAzureRuntimes(anyString(), any[UUID])
+  }
+
+  it should "poll and return true when all runtimes are in the error state" in {
+    val erroredRuntimeResponse = new ListRuntimeResponse()
+    erroredRuntimeResponse.setStatus(ClusterStatus.ERROR)
+    val leoDAO: MockLeonardoDAO = Mockito.spy(new MockLeonardoDAO() {
+      override def listAzureRuntimes(token: String, workspaceId: UUID): Seq[ListRuntimeResponse] =
+        Seq(erroredRuntimeResponse, erroredRuntimeResponse)
+    })
+
+    val action = new LeonardoResourceDeletionAction(leoDAO)
+
+    Await.result(action.pollRuntimeDeletion(azureWorkspace, ctx), Duration.Inf) shouldBe true
+    verify(leoDAO).listAzureRuntimes(anyString(), any[UUID])
+  }
+
 }