Small tweaks

Author: tw4l

backend/btrixcloud/crawls.py

@@ -365,8 +365,8 @@ async def get_active_crawls(self, oid: UUID, limit: int) -> list[str]:
         res_list = await res.to_list()
         return [res["_id"] for res in res_list]
 
-    async def get_active_crawls_size(self, oid: UUID) -> int:
-        """get size of all active (running, waiting, paused) crawls"""
+    async def get_active_crawls_pending_size(self, oid: UUID) -> int:
+        """get pending size of all active (running, waiting, paused) crawls"""
         cursor = self.crawls.aggregate(
             [
                 {"$match": {"state": {"$in": RUNNING_AND_WAITING_STATES}, "oid": oid}},

backend/btrixcloud/operator/crawls.py

@@ -396,8 +396,8 @@ async def sync_crawls(self, data: MCSyncData):
         if status.pagesFound < status.desiredScale:
             status.desiredScale = max(1, status.pagesFound)
 
-        # paused and shut down pods if size is <= 4096 (empty dir)
-        # paused_at is set state is a valid paused state
+        # paused and shut down pods if size is <= 4096 (empty dir),
+        # paused_at is set, and state is a valid paused state
         is_paused = (
             bool(crawl.paused_at)
             and status.sizePending <= 4096
@@ -1463,14 +1463,11 @@ async def is_crawl_stopping(
 
         # pause crawl if storage quota is reached
         if org.quotas.storageQuota:
-            # Make sure to account for already-uploaded WACZs from active crawls
-            # that are or previously were paused, which are already accounted for
-            # in the org storage stats
-            active_crawls_total_size = await self.crawl_ops.get_active_crawls_size(
-                crawl.oid
+            # include not-yet-uploaded pending data from all active crawls
+            active_crawls_pending_size = (
+                await self.crawl_ops.get_active_crawls_pending_size(crawl.oid)
             )
-
-            if self.org_ops.storage_quota_reached(org, active_crawls_total_size):
+            if self.org_ops.storage_quota_reached(org, active_crawls_pending_size):
                 return self.request_pause_crawl("paused_storage_quota_reached", crawl)
 
         # pause crawl if execution time quota is reached