Merge pull request #61 from MaloneGod/patch-3

Update the visited list

Author: REMitchell

chapter16/multiprocess_queue.py

@@ -21,6 +21,8 @@ def task_delegator(taskQueue, foundUrlsQueue):
             for link in links:
                 #Add new link to the taskQueue
                 taskQueue.put(link)
+                #Add new link to the visited list
+                visited.append(link)
 
 def get_links(bsObj):
     links = bsObj.find('div', {'id':'bodyContent'}).find_all('a', href=re.compile('^(/wiki/)((?!:).)*$'))
@@ -51,4 +53,4 @@ def scrape_article(taskQueue, foundUrlsQueue):
 processes.append(Process(target=scrape_article, args=(taskQueue, foundUrlsQueue,)))
 
 for p in processes:
-    p.start()
+    p.start()