Ver 3

subodhjena · Dec 30, 2013 · a7a571f · a7a571f
1 parent 07c0da3
commit a7a571f
Show file tree

Hide file tree

Showing 31 changed files with 128 additions and 33 deletions.
diff --git a/App.config b/App.config
@@ -7,7 +7,7 @@
   <abot>
     <crawlBehavior 
       maxConcurrentThreads="20" 
-      maxPagesToCrawl="100000000"
+      maxPagesToCrawl="10000000"
       maxPagesToCrawlPerDomain="0" 
       maxPageSizeInBytes="0"
       userAgentString="Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; abot v@ABOTASSEMBLYVERSION@ http://code.google.com/p/abot)" 

diff --git a/CrawlerDB.cs b/CrawlerDB.cs
@@ -6,24 +6,25 @@ namespace FileCrawler
 {
     public class CrawlerDB
     {
-        public const string DB_CONN_STRING = "data source=SUBODH;initial catalog=DownloadCrawler;Integrated security=true";
+        public const string DB_CONN_STRING = "data source=SUBODH;initial catalog=FileCrawler;Integrated security=true";
 
-        public void SaveFileURLToDB(String hostName, string fileType, string fileUrl)
+        public void SaveFileURLToDB(String tableName, String hostName, string fileType, string fileDescription,string fileUrl)
         {
             using (SqlConnection con = new SqlConnection(DB_CONN_STRING))
             {
                 con.Open();
                 try
                 {
-                    using (SqlCommand command = new SqlCommand("INSERT INTO [DownloadCrawler].[dbo].[CrawledFiles] VALUES(@host, @file, @url, @credt,@crename,@upddt,@updnm)", con))
+                    using (SqlCommand command = new SqlCommand("INSERT INTO [FileCrawler].[dbo].[" + tableName + "] VALUES(@hostName,@fileType,@fileDescription,@url,@createdDt,@createdByName,@updatedDt,@updatedByName)", con))
                     {
-                        command.Parameters.Add(new SqlParameter("host", hostName));
-                        command.Parameters.Add(new SqlParameter("file", fileType));
+                        command.Parameters.Add(new SqlParameter("hostName", hostName));
+                        command.Parameters.Add(new SqlParameter("fileType", fileType));
+                        command.Parameters.Add(new SqlParameter("fileDescription", fileDescription));
                         command.Parameters.Add(new SqlParameter("url", fileUrl));
-                        command.Parameters.Add(new SqlParameter("credt", DateTime.Now.ToString()));
-                        command.Parameters.Add(new SqlParameter("crename", "Subodh"));
-                        command.Parameters.Add(new SqlParameter("upddt", DBNull.Value));
-                        command.Parameters.Add(new SqlParameter("updnm", DBNull.Value));
+                        command.Parameters.Add(new SqlParameter("createdDt", DateTime.Now.ToString()));
+                        command.Parameters.Add(new SqlParameter("createdByName", "Subodh"));
+                        command.Parameters.Add(new SqlParameter("updatedDt", DBNull.Value));
+                        command.Parameters.Add(new SqlParameter("updatedByName", DBNull.Value));
                         command.ExecuteNonQuery();
                     }
                 }

diff --git a/FileCrawler.csproj b/FileCrawler.csproj
@@ -75,13 +75,39 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="CrawlerDB.cs" />
+    <Compile Include="FileTypes.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="App.config">
       <SubType>Designer</SubType>
     </None>
+    <None Include="FileTypes\3DImageFiles.csv" />
+    <None Include="FileTypes\AudioFiles.csv" />
+    <None Include="FileTypes\BackupFiles.csv" />
+    <None Include="FileTypes\CADFiles.csv" />
+    <None Include="FileTypes\CompressedFiles.csv" />
+    <None Include="FileTypes\DataFiles.csv" />
+    <None Include="FileTypes\DatabaseFiles.csv" />
+    <None Include="FileTypes\DeveloperFiles.csv" />
+    <None Include="FileTypes\DiskImageFiles.csv" />
+    <None Include="FileTypes\EncodedFiles.csv" />
+    <None Include="FileTypes\ExecutableFiles.csv" />
+    <None Include="FileTypes\FontFiles.csv" />
+    <None Include="FileTypes\GameFiles.csv" />
+    <None Include="FileTypes\GISFiles.csv" />
+    <None Include="FileTypes\MiscFiles.csv" />
+    <None Include="FileTypes\PageLayoutFiles.csv" />
+    <None Include="FileTypes\PluginFiles.csv" />
+    <None Include="FileTypes\RasterImageFiles.csv" />
+    <None Include="FileTypes\SettingsFiles.csv" />
+    <None Include="FileTypes\SpreadsheetFiles.csv" />
+    <None Include="FileTypes\SystemFiles.csv" />
+    <None Include="FileTypes\TextFiles.csv" />
+    <None Include="FileTypes\VectorImageFiles.csv" />
+    <None Include="FileTypes\VideoFiles.csv" />
+    <None Include="FileTypes\WebFiles.csv" />
   </ItemGroup>
   <ItemGroup>
     <BootstrapperPackage Include=".NETFramework,Version=v4.5">

diff --git a/FileCrawler.sql b/FileCrawler.sql
diff --git a/FileTypes.cs b/FileTypes.cs
@@ -0,0 +1,42 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace FileCrawler
+{
+    class FileTypes
+    {
+        List<String> fileList = new List<string>();
+        static StreamReader reader;
+
+        public List<String> GetFileTypesToDownlaod(String p)
+        {
+            string[] files = Directory.GetFiles(p);
+            foreach (String filePath in files)
+            {
+                ReadCSVToOne(filePath);
+            }
+            return fileList;
+        }
+
+        private void ReadCSVToOne(String filePath)
+        {
+            int counter = 0;
+            string line;
+
+            // Read the file and display it line by line.
+            reader = new System.IO.StreamReader(filePath);
+            while ((line = reader.ReadLine()) != null)
+            {
+                String fileName = Path.GetFileName(filePath).ToString().Substring(0, Path.GetFileName(filePath).ToString().Length-4);
+                fileList.Add(line+","+fileName);
+                counter++;
+            }
+
+            reader.Close();
+        }
+    }
+}
diff --git a/FileTypes/3D Image Files.csv → FileTypes/3DImageFiles.csv b/FileTypes/3D Image Files.csv → FileTypes/3DImageFiles.csv
diff --git a/FileTypes/Audio Files.csv → FileTypes/AudioFiles.csv b/FileTypes/Audio Files.csv → FileTypes/AudioFiles.csv
diff --git a/FileTypes/Backup Files.csv → FileTypes/BackupFiles.csv b/FileTypes/Backup Files.csv → FileTypes/BackupFiles.csv
diff --git a/FileTypes/CAD Files.csv → FileTypes/CADFiles.csv b/FileTypes/CAD Files.csv → FileTypes/CADFiles.csv
diff --git a/FileTypes/Compressed Files.csv → FileTypes/CompressedFiles.csv b/FileTypes/Compressed Files.csv → FileTypes/CompressedFiles.csv
diff --git a/FileTypes/Data Files.csv → FileTypes/DataFiles.csv b/FileTypes/Data Files.csv → FileTypes/DataFiles.csv
diff --git a/FileTypes/Database Files.csv → FileTypes/DatabaseFiles.csv b/FileTypes/Database Files.csv → FileTypes/DatabaseFiles.csv
diff --git a/FileTypes/Developer Files.csv → FileTypes/DeveloperFiles.csv b/FileTypes/Developer Files.csv → FileTypes/DeveloperFiles.csv
diff --git a/FileTypes/Disk Image Files.csv → FileTypes/DiskImageFiles.csv b/FileTypes/Disk Image Files.csv → FileTypes/DiskImageFiles.csv
diff --git a/FileTypes/Encoded Files.csv → FileTypes/EncodedFiles.csv b/FileTypes/Encoded Files.csv → FileTypes/EncodedFiles.csv
diff --git a/FileTypes/Executable Files.csv → FileTypes/ExecutableFiles.csv b/FileTypes/Executable Files.csv → FileTypes/ExecutableFiles.csv
diff --git a/FileTypes/Font Files.csv → FileTypes/FontFiles.csv b/FileTypes/Font Files.csv → FileTypes/FontFiles.csv
diff --git a/FileTypes/GIS Files.csv → FileTypes/GISFiles.csv b/FileTypes/GIS Files.csv → FileTypes/GISFiles.csv
diff --git a/FileTypes/Game Files.csv → FileTypes/GameFiles.csv b/FileTypes/Game Files.csv → FileTypes/GameFiles.csv
diff --git a/FileTypes/Misc Files.csv → FileTypes/MiscFiles.csv b/FileTypes/Misc Files.csv → FileTypes/MiscFiles.csv
diff --git a/FileTypes/Page Layout Files.csv → FileTypes/PageLayoutFiles.csv b/FileTypes/Page Layout Files.csv → FileTypes/PageLayoutFiles.csv
diff --git a/FileTypes/Plugin Files.csv → FileTypes/PluginFiles.csv b/FileTypes/Plugin Files.csv → FileTypes/PluginFiles.csv
diff --git a/FileTypes/Raster Image Files.csv → FileTypes/RasterImageFiles.csv b/FileTypes/Raster Image Files.csv → FileTypes/RasterImageFiles.csv
diff --git a/FileTypes/Settings Files.csv → FileTypes/SettingsFiles.csv b/FileTypes/Settings Files.csv → FileTypes/SettingsFiles.csv
diff --git a/FileTypes/Spreadsheet Files.csv → FileTypes/SpreadsheetFiles.csv b/FileTypes/Spreadsheet Files.csv → FileTypes/SpreadsheetFiles.csv
diff --git a/FileTypes/System Files.csv → FileTypes/SystemFiles.csv b/FileTypes/System Files.csv → FileTypes/SystemFiles.csv
diff --git a/FileTypes/Text Files.csv → FileTypes/TextFiles.csv b/FileTypes/Text Files.csv → FileTypes/TextFiles.csv
diff --git a/FileTypes/Vector Image Files.csv → FileTypes/VectorImageFiles.csv b/FileTypes/Vector Image Files.csv → FileTypes/VectorImageFiles.csv
diff --git a/FileTypes/Video Files.csv → FileTypes/VideoFiles.csv b/FileTypes/Video Files.csv → FileTypes/VideoFiles.csv
diff --git a/FileTypes/Web Files.csv → FileTypes/WebFiles.csv b/FileTypes/Web Files.csv → FileTypes/WebFiles.csv
diff --git a/Program.cs b/Program.cs
@@ -14,12 +14,22 @@ namespace FileCrawler
 {
     class Program
     {
-        static String webURL = @"http://www.songspk.name/";
-        static String filterTxt = ".zip";
+        #region "Fields and Object Declaration"
+
+        static String webURL = @"http://ebooks.allfree-stuff.com/";
+        static String fileTypePath = @"C:\Users\Subodhlc\Documents\Visual Studio 2012\Projects\FileCrawler\FileCrawler\FileTypes";
         static CrawlerDB crawalerDatabase = new CrawlerDB();
-
+        static FileTypes fileTyp = new FileTypes();
+        static List<String> filters;
+
+        #endregion
+
+
         static void Main(string[] args)
         {
+            //Will Get the FileTypes to Download
+            filters = fileTyp.GetFileTypesToDownlaod(fileTypePath);
+
             //Will use app.config for confguration
             PoliteWebCrawler crawler = new PoliteWebCrawler();
 
@@ -48,7 +58,7 @@ static void crawler_ProcessPageCrawlCompleted(object sender, PageCrawlCompletedA
                 SaveURLFail(crawledPage.Uri.AbsoluteUri.ToString());
             else
                 SaveURLSuccess(crawledPage.Uri.AbsoluteUri.ToString());
-                            
+
             if (string.IsNullOrEmpty(crawledPage.RawContent))
                 SaveURLNoContent(crawledPage.Uri.AbsoluteUri.ToString());
         }
@@ -69,42 +79,58 @@ static void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e
         private static void SaveURLSuccess(string p)
         {
             Console.WriteLine("Shubh :Crawl of page succeeded {0}", p);
-            if (p.Contains(filterTxt) && Uri.IsWellFormedUriString(p, UriKind.RelativeOrAbsolute))
-            {
-                crawalerDatabase.SaveFileURLToDB(webURL, filterTxt, p);
-            }
+            WriteToDB(p);
         }
         private static void SaveURLFail(string p)
         {
             Console.WriteLine("Shubh :Crawl of page Failed {0}", p);
-            if (p.Contains(filterTxt) && Uri.IsWellFormedUriString(p, UriKind.RelativeOrAbsolute))
-            {
-                crawalerDatabase.SaveFileURLToDB(webURL, filterTxt, p);
-            }
+            WriteToDB(p);
         }
         private static void SavePageLinksCrawlDisallowed(string p)
         {
             Console.WriteLine("Shubh :Page Links Craw lDisallowed {0}", p);
-            if (p.Contains(filterTxt) && Uri.IsWellFormedUriString(p, UriKind.RelativeOrAbsolute))
-            {
-                crawalerDatabase.SaveFileURLToDB(webURL, filterTxt, p);
-            }
+            WriteToDB(p);
         }
         private static void SaveURLNoContent(string p)
         {
             Console.WriteLine("Shubh :Crawl of page had no content {0}", p);
-            if (p.Contains(filterTxt) && Uri.IsWellFormedUriString(p, UriKind.RelativeOrAbsolute))
-            {
-                crawalerDatabase.SaveFileURLToDB(webURL, filterTxt, p);
-            }
+            WriteToDB(p);
         }
         private static void SavePageCrawlDisallowed(string p)
         {
             Console.WriteLine("Shubh :Crawl of page Not allowed {0}", p);
-            if (p.Contains(filterTxt) && Uri.IsWellFormedUriString(p, UriKind.RelativeOrAbsolute))
+            WriteToDB(p);
+        }
+
+        //DB Writes
+        private static void WriteToDB(string p)
+        {
+            try
             {
-                crawalerDatabase.SaveFileURLToDB(webURL, filterTxt, p);
+
+                foreach (String filter in filters)
+                {
+                    String[] splitFilters = filter.Split(',');
+                    String filterFileType = splitFilters[0];
+                    String filterFileDescription = splitFilters[1];
+                    String filterFileTable = splitFilters[2];
+
+                    if (p.Trim().EndsWith(filterFileType) && Uri.IsWellFormedUriString(p, UriKind.RelativeOrAbsolute))
+                    {
+                        crawalerDatabase.SaveFileURLToDB(filterFileTable, webURL, filterFileType, filterFileDescription, p);
+                        Console.WriteLine("Wrote :" + p + " to :" + filterFileTable);
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                System.Console.WriteLine("**************************************");
+                System.Console.WriteLine("WRITING: ERROR DESCRIPTION");
+                System.Console.WriteLine("  Error Message:" + ex.Message);
+                System.Console.WriteLine("  Stack Trace  :" + ex.StackTrace);
+                System.Console.WriteLine("**************************************");
             }
         }
+
     }
-}
+}