(fix): Bunkrr child item fix and improve scraping performance

.gitignore

@@ -13,6 +13,18 @@
 
 # Python cache
 __pycache__
+/.pytest_cache/
+/.ruff_cache/
+/pytest-cache-files-*/
+
+# Local crawler debug/temp artifacts
+/bunkrr_*.txt
+/output_bunkrr*.txt
+/check_db.py
+/tmp_*/
+/tmp_*.html
+/tmp_*.js
+
 # venv
 .venv
 venv
@@ -23,6 +35,7 @@ venv
 
 # Compiled package
 /dist/
+/scripts/release/
 
 # MacOS system files
 **.DS_Store

cyberdrop_dl/crawlers/__init__.py

@@ -36,6 +36,7 @@
 from .fapello import FapelloCrawler
 from .fileditch import FileditchCrawler
 from .files_vc import FilesVcCrawler
+from .filester import FilesterCrawler
 from .flickr import FlickrCrawler
 from .flugel_anime import FlugelAnimeCrawler
 from .fourchan import FourChanCrawler
@@ -133,6 +134,7 @@
 from .twitter_images import TwimgCrawler
 from .twpornstars import TwPornstarsCrawler
 from .txxx import TXXXCrawler
+from .upload_ee import UploadEECrawler
 from .upornia import UPorniaCrawler
 from .vbulletin import VBULLETIN_CRAWLERS
 from .vipr_dot_im import ViprImCrawler

cyberdrop_dl/crawlers/_chevereto.py

@@ -127,7 +127,7 @@ async def profile(self, scrape_item: ScrapeItem, *, albums: bool = False) -> Non
                 for _, sub_album in self.iter_children(scrape_item, soup, Selector.ITEM):
                     self.create_task(self.run(sub_album))
 
-                return
+                continue
 
             self._iter_album_files(scrape_item, soup)
 

cyberdrop_dl/crawlers/_forum.py

@@ -567,8 +567,9 @@ async def get_absolute_link(self, link: str | AbsoluteHttpURL) -> AbsoluteHttpUR
 
     @error_handling_wrapper
     async def resolve_confirmation_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL | None:
-        if url := link.query.get("url"):
-            url = base64.b64decode(url).decode("utf-8")
+        if url := link.query.get("url") or link.query.get("to"):
+            padding = -len(url) % 4
+            url = base64.urlsafe_b64decode(url + "=" * padding).decode("utf-8")
             if url.startswith("https://"):
                 return self.parse_url(url)
 
@@ -692,7 +693,9 @@ def clean_link_str(link: str) -> str:
 
 
 def is_confirmation_link(link: AbsoluteHttpURL) -> bool:
-    return "masked" in link.parts or "link-confirmation" in link.path
+    return (
+        "masked" in link.parts or "link-confirmation" in link.path or ("redirect" in link.parts and "to" in link.query)
+    )
 
 
 def check_post_id(init_post_id: int | None, current_post_id: int, scrape_single_forum_post: bool) -> tuple[bool, bool]: