Remove ^ from Regex for Wiki links

ACPK · web-flow · commit 217b65461f75 · 2023-02-06T10:46:27.000-05:00
The links being scraped are now absolute (full) URLs. The rule should look for links that contain `/wiki/` rather than start with `/wiki/`. Ex: https://en.wikipedia.org/wiki/Benevolent_dictator_for_life rather than /wiki/Benevolent_dictator_for_life'
diff --git a/Chapter05_Scrapy/wikiSpider/wikiSpider/articlesMoreRules.py b/Chapter05_Scrapy/wikiSpider/wikiSpider/articlesMoreRules.py
@@ -6,7 +6,7 @@ class ArticleSpider(CrawlSpider):
     allowed_domains = ['wikipedia.org']
     start_urls = ['https://en.wikipedia.org/wiki/Benevolent_dictator_for_life']
     rules = [
-        Rule(LinkExtractor(allow='^(/wiki/)((?!:).)*$'), callback='parse_items', follow=True, cb_kwargs={'is_article': True}),
+        Rule(LinkExtractor(allow='(/wiki/)((?!:).)*$'), callback='parse_items', follow=True, cb_kwargs={'is_article': True}),
         Rule(LinkExtractor(allow='.*'), callback='parse_items', cb_kwargs={'is_article': False})
     ]
 

Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@ class ArticleSpider(CrawlSpider):`
`6`	`6`	`allowed_domains = ['wikipedia.org']`
`7`	`7`	`start_urls = ['https://en.wikipedia.org/wiki/Benevolent_dictator_for_life']`
`8`	`8`	`rules = [`
`9`		`- Rule(LinkExtractor(allow='^(/wiki/)((?!:).)*$'), callback='parse_items', follow=True, cb_kwargs={'is_article': True}),`
	`9`	`+ Rule(LinkExtractor(allow='(/wiki/)((?!:).)*$'), callback='parse_items', follow=True, cb_kwargs={'is_article': True}),`
`10`	`10`	`Rule(LinkExtractor(allow='.*'), callback='parse_items', cb_kwargs={'is_article': False})`
`11`	`11`	`]`
`12`	`12`