2023.11.24.0

Scheduler: handle scheduler change error (collection was modified)
SchedulerEditorForm: add scheduler name to form title
DownloadProgress: fix disposing error when some objects are already null
API.Reddit: add special notification for error 429
API.Twitter: handle JSON deserialization error
Porn sites: fix incorrect parsing of search queries
YouTube: path not set when adding array to download

Author: AAndyProgram

Changelog.md

@@ -1,3 +1,27 @@
+# 2023.11.24.0
+
+*2023-11-24*
+
+For those of you who use TikTok, I recommend updating [TikTok plugin](https://github.com/bashonly/yt-dlp-TTUser) to the latest version using [these instructions](https://github.com/AAndyProgram/SCrawler/wiki/Settings#how-to-install-yt-dlp-ttuser-plugin).
+
+- Added
+  - Automation: manual task option
+  - Scheduler: add scheduler name to form title
+  - Feeds: update when users' location and/or basic information changes
+  - Reddit: special notification for error 429
+  - TikTok: ID, username and friendly name extraction from data
+  - TikTok: new option `Use video date as file date`
+  - YouTube: absolute path for a single playlist
+- Updated
+  - yt-dlp up to version 2023.11.16
+- Fixed
+  - Scheduler: scheduler change error
+  - Twitter: JSON deserialization error
+  - xHamster, XVideos, PornHub, ThisVid: incorrect parsing of search queries
+  - YouTube: the file name is not changed manually
+  - YouTube: path not set when adding array to download
+  - Minor bugs
+
 # 2023.11.17.0
 
 *2023-11-17*

SCrawler.PluginProvider/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2023.11.17.0")>
-<Assembly: AssemblyFileVersion("2023.11.17.0")>
+<Assembly: AssemblyVersion("2023.11.24.0")>
+<Assembly: AssemblyFileVersion("2023.11.24.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler.YouTube/Downloader/VideoListForm.vb

@@ -278,6 +278,17 @@ Namespace DownloadObjects.STDownloader
                                             .IsMusic = containers.Any(Function(cc) cc.IsMusic)
                                         }
                                         c.Elements.AddRange(containers)
+                                        Dim path$ = c.Elements(0).File.PathWithSeparator
+                                        For Each list As List(Of String) In {
+                                            c.Elements.Select(Function(cc) cc.UserTitle).ListWithRemove(Function(cc) cc.IsEmptyString).ListIfNothing,
+                                            c.Elements.Select(Function(cc) cc.PlaylistTitle).ListWithRemove(Function(cc) cc.IsEmptyString).ListIfNothing
+                                        }
+                                            If list.Count > 0 AndAlso
+                                               (list.Count = 1 OrElse
+                                               ListAddList(Nothing, list, LAP.NotContainsOnly, EDP.ReturnValue).ListIfNothing.Count = 1) Then _
+                                               path &= $"{list(0)}\"
+                                        Next
+                                        c.File = path
                                     End If
                                 End If
                             End With
@@ -450,12 +461,16 @@ Namespace DownloadObjects.STDownloader
             UpdateLogButton()
         End Sub
         Protected Sub AddToDownload(ByRef Item As MediaItem, ByVal RunThread As Boolean)
-            Dim hc% = Item.MyContainer.GetHashCode
-            If MyJob.Count = 0 OrElse Not MyJob.Items.Exists(Function(i) i.MyContainer.GetHashCode = hc) Then
-                MyJob.Add(Item)
-                Item.AddToQueue()
-                If RunThread Then StartDownloading()
-            End If
+            Try
+                Dim hc% = Item.MyContainer.GetHashCode
+                If MyJob.Count = 0 OrElse Not MyJob.Items.Exists(Function(i) i.MyContainer.GetHashCode = hc) Then
+                    MyJob.Add(Item)
+                    Item.AddToQueue()
+                    If RunThread Then StartDownloading()
+                End If
+            Catch ex As Exception
+                ErrorsDescriber.Execute(EDP.SendToLog, ex, "[VideoListForm.AddToDownload]")
+            End Try
         End Sub
         Private Sub StartDownloading()
             If Not MyJob.Working And MyJob.Count > 0 Then

SCrawler.YouTube/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2023.11.17.0")>
-<Assembly: AssemblyFileVersion("2023.11.17.0")>
+<Assembly: AssemblyVersion("2023.11.24.0")>
+<Assembly: AssemblyFileVersion("2023.11.24.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2023.11.17.0")>
-<Assembly: AssemblyFileVersion("2023.11.17.0")>
+<Assembly: AssemblyVersion("2023.11.24.0")>
+<Assembly: AssemblyFileVersion("2023.11.24.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/API/Base/UserDataBase.vb

@@ -1729,8 +1729,6 @@ BlockNullPicture:
                                         DownloadContentDefault_PostProcessing(v, f, Token)
                                         dCount += 1
                                     Catch woex As OperationCanceledException When Token.IsCancellationRequested
-                                        'TODELETE: UserDataBase.DownloadContentDefault: remove file when 'OperationCanceledException'
-                                        'If f.Exists Then f.Delete(,, EDP.SendToLog)
                                         __deleteFile.Invoke(f, v.URL_BASE)
                                         v.State = UStates.Missing
                                         v.Attempts += 1

SCrawler/API/JustForFans/M3U8.vb

@@ -126,70 +126,6 @@ Namespace API.JustForFans
                                         $"API.JustForFans.M3U8.GetFiles({IIf(IsAudio, "audio", "video")}):{vbCr}URL: {URL}{vbCr}File: {File}")
             End Try
         End Sub
-        'TODELETE: JFF.M3U8.GetFiles_OLD 20231008
-        'Private Sub GetFiles_OLD(ByVal URL As String, ByRef File As SFile, ByVal IsAudio As Boolean)
-        '    Try
-        '        Dim r$ = Responser.GetResponse(URL)
-        '        If Not r.IsEmptyString Then
-        '            Dim data As List(Of RegexMatchStruct) = RegexFields(Of RegexMatchStruct)(r, {REGEX_PLS_FILES}, {1, 2}, EDP.ReturnValue)
-        '            If data.ListExists Then
-        '                Dim appender$ = URL.Replace(URL.Split("/").LastOrDefault, String.Empty)
-        '                With (From d As RegexMatchStruct In data
-        '                      Where Not d.Arr(0).IfNullOrEmpty(d.Arr(1)).IsEmptyString
-        '                      Select M3U8Base.CreateUrl(appender, d.Arr(0).IfNullOrEmpty(d.Arr(1)).Trim)).ToList
-        '                    If .ListExists Then
-        '                        File = $"{Cache.RootDirectory.PathWithSeparator}{IIf(IsAudio, "AUDIO.aac", "VIDEO.mp4")}"
-        '                        Dim tmpCache As CacheKeeper = Cache.NewInstance
-        '                        Dim tmpFile As SFile = .Item(0)
-        '                        If tmpFile.Extension.IsEmptyString Then tmpFile.Extension = "ts"
-        '                        tmpFile.Path = tmpCache.RootDirectory.Path
-        '                        tmpFile.Separator = "\"
-
-        '                        Dim cFile As SFile = tmpFile
-        '                        cFile.Name = "all"
-
-        '                        tmpCache.Validate()
-
-        '                        Using bat As New TextSaver
-        '                            Using b As New BatchExecutor(True) With {.Encoding = Settings.CMDEncoding}
-        '                                AddHandler b.OutputDataReceived, AddressOf Batch_OutputDataReceived
-        '                                bat.AppendLine($"chcp {BatchExecutor.UnicodeEncoding}")
-        '                                bat.AppendLine(BatchExecutor.GetDirectoryCommand(tmpCache))
-        '                                ProgressChangeMax(.Count * 2 + 1)
-        '                                Using w As New WebClient
-        '                                    For i = 0 To .Count - 1
-        '                                        tmpFile.Name = $"ConPart_{i}"
-        '                                        Thrower.ThrowAny()
-        '                                        'Responser.DownloadFile(.Item(i), tmpFile)
-        '                                        w.DownloadFile(.Item(i), tmpFile)
-        '                                        ProgressPerform()
-        '                                        tmpCache.AddFile(tmpFile, True)
-        '                                        bat.AppendLine($"type {tmpFile.File} >> {cFile.File}")
-        '                                    Next
-        '                                End Using
-
-        '                                bat.AppendLine($"""{Settings.FfmpegFile}"" -i {cFile.File} -c copy ""{File}""")
-
-        '                                Dim batFile As SFile = bat.SaveAs($"{tmpCache.RootDirectory.PathWithSeparator}command.bat")
-
-        '                                b.Execute($"""{batFile}""")
-
-        '                                If Not File.Exists Then File = Nothing
-        '                            End Using
-        '                        End Using
-        '                    End If
-        '                End With
-        '            End If
-        '        End If
-        '    Catch oex As OperationCanceledException
-        '        Throw oex
-        '    Catch dex As ObjectDisposedException
-        '        Throw dex
-        '    Catch ex As Exception
-        '        ErrorsDescriber.Execute(EDP.SendToLog + EDP.ThrowException, ex,
-        '                                $"API.JustForFans.M3U8.GetFiles({IIf(IsAudio, "audio", "video")}):{vbCr}URL: {URL}{vbCr}File: {File}")
-        '    End Try
-        'End Sub
         Private Async Sub Batch_OutputDataReceived(ByVal Sender As Object, ByVal e As DataReceivedEventArgs)
             Await Task.Run(Sub() If Not e.Data.IsEmptyString AndAlso e.Data.Contains("] Opening") Then ProgressPerform())
         End Sub

SCrawler/API/JustForFans/UserData.vb

@@ -168,8 +168,6 @@ Namespace API.JustForFans
 #Region "Initializer"
         Friend Sub New()
             UseInternalM3U8Function = True
-            'TODELETE: UseResponserClient 20231008
-            'UseResponserClient = True
         End Sub
 #End Region
 #Region "Download functions"

SCrawler/API/PornHub/UserData.vb

@@ -423,30 +423,28 @@ Namespace API.PornHub
                                             newLastPageIDs.Add(uv.ID)
                                             If Not _TempPostsList.Contains(uv.ID) Then
                                                 _TempPostsList.Add(uv.ID)
+                                                newPostsFound = True
                                                 Return False
                                             ElseIf SessionPosts.Count > 0 AndAlso SessionPosts.Contains(uv.id) Then
                                                 prevPostsFound = True
-                                                If pageRepeatSet Then pageRepeatSet = False : _PageVideosRepeat -= 1
                                                 Return True
                                             Else
-                                                'TODELETE: PornHub old validating
-                                                'If Not SessionPosts.Contains(uv.ID) Then nonLastPageDetected = True
                                                 If Not pageRepeatSet And Not newPostsFound Then pageRepeatSet = True : _PageVideosRepeat += 1
                                                 'Debug.WriteLine($"[REMOVED]: {uv.Title}")
                                                 Return True
                                             End If
                                         End Function)
                             'Debug.WriteLineIf(l.Count > 0, l.Select(Function(ll) ll.Title).ListToString(vbNewLine))
+                            If prevPostsFound And Not pageRepeatSet And Not newPostsFound Then pageRepeatSet = True : _PageVideosRepeat += 1
+                            If prevPostsFound And newPostsFound And pageRepeatSet Then _PageVideosRepeat -= 1
                             If l.Count > 0 Then _TempMediaList.ListAddList(l.Select(Function(uv) uv.ToUserMedia(specFolder)))
                             SessionPosts.ListAddList(newLastPageIDs, LNC)
                             newLastPageIDs.Clear()
-                            'TODELETE: PornHub old validating
-                            'If l.Count > 0 AndAlso (l.Count = lBefore Or Not nonLastPageDetected) AndAlso
-                            '   Not (limit > 0 And _TempMediaList.Count >= limit) Then tryNextPage = True
 
                             If limit > 0 And _TempMediaList.Count >= limit Then Exit Sub
-                            If (Not IsUser And prevPostsFound And Not newPostsFound And Page < 1000) Or
-                               (Not cBefore = _TempMediaList.Count And (IsUser Or Page < 1000)) Then tryNextPage = True
+                            If _PageVideosRepeat < 2 And
+                               ((Not IsUser And prevPostsFound And Not newPostsFound And Page < 1000) Or
+                               (Not cBefore = _TempMediaList.Count And (IsUser Or Page < 1000))) Then tryNextPage = True
 
                             l.Clear()
                         End If

SCrawler/API/Reddit/SiteSettings.vb

@@ -50,6 +50,12 @@ Namespace API.Reddit
         Friend ReadOnly Property UseCookiesForTimelines As PropertyValue
         <PropertyOption(ControlText:=DeclaredNames.SavedPostsUserNameCaption, ControlToolTip:=DeclaredNames.SavedPostsUserNameToolTip, IsAuth:=True), PXML, PClonable(Clone:=False)>
         Friend ReadOnly Property SavedPostsUserName As PropertyValue
+        Friend ReadOnly Property CredentialsExists As Boolean
+            Get
+                Return {AuthUserName.Value, AuthPassword.Value, ApiClientID.Value, ApiClientSecret.Value}.All(Function(v$) Not v.IsEmptyString)
+            End Get
+        End Property
+
 #End Region
 #Region "Other"
         <PropertyOption(ControlText:="Use M3U8", ControlToolTip:="Use M3U8 or mp4 for Reddit videos", IsAuth:=False), PXML, PClonable>
@@ -238,8 +244,7 @@ Namespace API.Reddit
             Return False
         End Function
         Private Function UpdateTokenIfRequired() As Boolean
-            If (CBool(UseTokenForTimelines.Value) Or CBool(UseTokenForSavedPosts.Value)) AndAlso
-               {AuthUserName.Value, AuthPassword.Value, ApiClientID.Value, ApiClientSecret.Value}.All(Function(v$) Not v.IsEmptyString) Then
+            If (CBool(UseTokenForTimelines.Value) Or CBool(UseTokenForSavedPosts.Value)) AndAlso CredentialsExists Then
                 If CDate(BearerTokenDateUpdate.Value).AddMinutes(TokenUpdateInterval.Value) <= Now Then Return UpdateToken()
             End If
             Return True