2024.4.26.0

Add 'CookieValueExtractorAttribute' and the ability to immediately populate fields with values that can be extracted from cookies
Feed: add the ability to load the last session of the current day (if it exists) as the current session after restarting SCrawler
UserSearchForm: include friendly name matches in search result
API.Xhamster: saved posts aren't downloading

Author: AAndyProgram

Changelog.md

@@ -1,3 +1,16 @@
+# 2024.4.26.0
+
+*2024-04-26*
+
+- Added
+  - Site settings: the values that can be extracted from cookies immediately populate fields
+  - Feed: ability to load the last session of the current day (if it exists) as the current session after restarting SCrawler
+  - Users search: include friendly name matches in search result
+- Updated
+  - gallery-dl up to version **1.26.9**
+- Fixed
+  - xHamster: **saved posts aren't downloading**
+
 # 2024.4.14.0
 
 *2024-04-14*

ProgramScreenshots/SettingsGlobalFeed.png

@@ -6,6 +6,7 @@
 '
 ' This program is distributed in the hope that it will be useful,
 ' but WITHOUT ANY WARRANTY
+Imports System.Runtime.CompilerServices
 Imports PersonalUtilities.Forms
 Imports PersonalUtilities.Functions.RegularExpressions
 Namespace API.Base
@@ -72,5 +73,12 @@ Namespace API.Base
                             $"Current query: [{CurrentQuery}]{vbCr}New query: [{NewQuery}]",
                             "Changing a query"}, vbExclamation,,, {"Process", "Cancel"}) = 0
         End Function
+        <Extension> Friend Function GetCookieValue(ByVal Cookies As IEnumerable(Of System.Net.Cookie), ByVal CookieName As String) As String
+            If Cookies.ListExists Then Return If(Cookies.FirstOrDefault(Function(c) c.Name.ToLower = CookieName.ToLower)?.Value, String.Empty) Else Return String.Empty
+        End Function
+        <Extension> Friend Function GetCookieValue(ByVal Cookies As IEnumerable(Of System.Net.Cookie), ByVal CookieName As String,
+                                                   ByVal PropName As String, ByVal PropNameComp As String) As String
+            Return If(PropName = PropNameComp, Cookies.GetCookieValue(CookieName), String.Empty)
+        End Function
     End Module
 End Namespace

SCrawler/API/Base/Declarations.vb

@@ -65,6 +65,10 @@ Namespace API.Instagram
         Friend Const Header_Platform_Verion As String = "Sec-Ch-Ua-Platform-Version"
         <PropertyOption(ControlText:="x-csrftoken", ControlToolTip:="Can be automatically extracted from cookies", IsAuth:=True, AllowNull:=True), ControlNumber(2), PClonable(Clone:=False)>
         Friend ReadOnly Property HH_CSRF_TOKEN As PropertyValue
+        <CookieValueExtractor(NameOf(HH_CSRF_TOKEN))>
+        Private Function GetValueFromCookies(ByVal PropName As String, ByVal c As CookieKeeper) As String
+            Return c.GetCookieValue(Header_CSRF_TOKEN_COOKIE, PropName, NameOf(HH_CSRF_TOKEN))
+        End Function
         <PropertyOption(ControlText:="x-ig-app-id", IsAuth:=True, AllowNull:=False), ControlNumber(3), PClonable(Clone:=False)>
         Friend ReadOnly Property HH_IG_APP_ID As PropertyValue
         <PropertyOption(ControlText:="x-asbd-id", IsAuth:=True, AllowNull:=True), ControlNumber(4), PClonable(Clone:=False)>
@@ -557,7 +561,7 @@ Namespace API.Instagram
                 If vals.Any(Function(v) Not v.ValueOld = v.ValueNew) OrElse
                    Not Responser.Cookies.ListEquals(____Cookies) Then HH_IG_WWW_CLAIM.Value = 0 : credentialsUpdated = True
                 If Responser.CookiesExists Then
-                    Dim csrf$ = If(Responser.Cookies.FirstOrDefault(Function(c) c.Name.StringToLower = Header_CSRF_TOKEN_COOKIE)?.Value, String.Empty)
+                    Dim csrf$ = GetValueFromCookies(NameOf(HH_CSRF_TOKEN), Responser.Cookies)
                     If Not csrf.IsEmptyString Then
                         If Not AEquals(Of String)(CStr(HH_CSRF_TOKEN.Value), csrf) Then credentialsUpdated = True
                         HH_CSRF_TOKEN.Value = csrf

SCrawler/API/Instagram/SiteSettings.vb

@@ -20,6 +20,10 @@ Namespace API.JustForFans
         Friend ReadOnly Property UserID As PropertyValue
         <PropertyOption, PXML, PClonable(Clone:=False)>
         Friend ReadOnly Property UserHash4 As PropertyValue
+        <CookieValueExtractor(NameOf(UserHash4))>
+        Private Function GetValueFromCookies(ByVal PropName As String, ByVal c As CookieKeeper) As String
+            Return c.GetCookieValue(UserHash4_CookieName, PropName, NameOf(UserHash4))
+        End Function
         <PropertyOption(ControlText:="Accept", ControlToolTip:="Header 'Accept'"), PClonable>
         Friend ReadOnly Property HeaderAccept As PropertyValue
         <PropertyOption(InheritanceName:=SettingsCLS.HEADER_DEF_UserAgent), PClonable, PXML(OnlyForChecked:=True)>
@@ -61,7 +65,7 @@ Namespace API.JustForFans
         Private Sub UpdateUserHash4()
             If Responser.CookiesExists Then
                 Dim hv_current$ = UserHash4.Value
-                Dim hv_cookie$ = If(Responser.Cookies.FirstOrDefault(Function(cc) cc.Name.ToLower = UserHash4_CookieName)?.Value, String.Empty)
+                Dim hv_cookie$ = GetValueFromCookies(NameOf(UserHash4), Responser.Cookies)
                 If Not hv_cookie.IsEmptyString And Not hv_cookie = hv_current And Responser.Cookies.Changed Then UserHash4.Value = hv_cookie
             End If
         End Sub

SCrawler/API/JustForFans/SiteSettings.vb

@@ -59,6 +59,16 @@ Namespace API.OnlyFans
                 Responser.UserAgent = Value
             End If
         End Sub
+        <CookieValueExtractor(NameOf(HH_USER_ID)), CookieValueExtractor(NameOf(HH_X_BC))>
+        Private Function GetValueFromCookies(ByVal PropName As String, ByVal c As CookieKeeper) As String
+            If c.ListExists Then
+                Select Case PropName
+                    Case NameOf(HH_USER_ID) : Return c.GetCookieValue("auth_id")
+                    Case NameOf(HH_X_BC) : Return c.GetCookieValue("fp")
+                End Select
+            End If
+            Return String.Empty
+        End Function
 #End Region
 #Region "Rules"
         <PXML("LastDateUpdated")> Private ReadOnly Property LastDateUpdated_XML As PropertyValue

SCrawler/API/OnlyFans/SiteSettings.vb

@@ -287,7 +287,7 @@ Namespace API.Pinterest
             End Function
         End Class
         Private Function GetDataFromGalleryDL(ByVal URL As String, ByVal IsBoardsRequested As Boolean, ByVal Token As CancellationToken) As List(Of String)
-            Dim command$ = $"gallery-dl --verbose --simulate "
+            Dim command$ = $"""{Settings.GalleryDLFile.File}"" --verbose --simulate "
             Try
                 If Not URL.IsEmptyString Then
                     If MySettings.CookiesNetscapeFile.Exists Then command &= $"--cookies ""{MySettings.CookiesNetscapeFile}"" "

SCrawler/API/Pinterest/UserData.vb

@@ -25,6 +25,10 @@ Namespace API.ThreadsNet
                 Return __HH_CSRF_TOKEN
             End Get
         End Property
+        <CookieValueExtractor(NameOf(HH_CSRF_TOKEN))>
+        Private Function GetValueFromCookies(ByVal PropName As String, ByVal c As CookieKeeper) As String
+            Return c.GetCookieValue(IG.Header_CSRF_TOKEN_COOKIE, PropName, NameOf(HH_CSRF_TOKEN))
+        End Function
         <PClonable> Protected ReadOnly __HH_IG_APP_ID As PropertyValue
         <PropertyOption(ControlText:="x-ig-app-id", AllowNull:=False, IsAuth:=True), ControlNumber(10)>
         Friend Overridable ReadOnly Property HH_IG_APP_ID As PropertyValue
@@ -195,7 +199,7 @@ Namespace API.ThreadsNet
         End Sub
         Friend Overrides Sub Update()
             If _SiteEditorFormOpened And Responser.CookiesExists Then
-                Dim csrf$ = If(Responser.Cookies.FirstOrDefault(Function(c) c.Name.StringToLower = IG.Header_CSRF_TOKEN_COOKIE)?.Value, String.Empty)
+                Dim csrf$ = GetValueFromCookies(NameOf(HH_CSRF_TOKEN), Responser.Cookies)
                 If Not csrf.IsEmptyString Then HH_CSRF_TOKEN.Value = csrf
                 If Not __Cookies Is Nothing AndAlso Not __Cookies.ListEquals(Responser.Cookies) Then DownloadData_Impl.Value = True
             End If

SCrawler/API/ThreadsNet/SiteSettings.vb

@@ -265,7 +265,12 @@ Namespace API.Xhamster
                 Dim checkLimit As Func(Of Boolean) = Function() limit > 0 And SearchPostsCount >= limit And IsVideo
 
                 If IsSavedPosts Then
-                    containerNodes.Add(If(IsVideo, {"favoriteVideoListComponent", "models"}, {"favoritesGalleriesAndPhotosCollection"}))
+                    If IsVideo Then
+                        containerNodes.Add({"favoriteVideoListComponent", "models"})
+                        containerNodes.Add({"favoriteVideoListComponent", "videoThumbProps"})
+                    Else
+                        containerNodes.Add({"favoritesGalleriesAndPhotosCollection"})
+                    End If
                 ElseIf Not SiteMode = SiteModes.Search Then
                     If IsVideo Then
                         containerNodes.Add({"trendingVideoListComponent", "models"})

SCrawler/API/Xhamster/UserData.vb

@@ -173,6 +173,7 @@ Namespace DownloadObjects
                 MENU_DOWN.Visible = OPT_SUBSCRIPTIONS.Checked
                 UpdateSettings()
                 FeedChangeMode(FeedModes.Current)
+                Downloader.FilesLoadLastSession()
                 RefillList(True, False)
                 .EndLoaderOperations(False)
             End With