-
Notifications
You must be signed in to change notification settings - Fork 29
/
Copy pathbad-crawler.rules
83 lines (83 loc) · 3.82 KB
/
bad-crawler.rules
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
(http.user_agent contains "80legs") or
(http.user_agent contains "Abonti") or
(http.user_agent contains "admantx") or
(http.user_agent contains "aipbot") or
(http.user_agent contains "AllSubmitter") or
(http.user_agent contains "Backlink") or (http.user_agent contains "backlink") or
(http.user_agent contains "Badass") or
(http.user_agent contains "Bigfoot") or
(http.user_agent contains "blexbot") or
(http.user_agent contains "Buddy") or
(http.user_agent contains "CherryPicker") or
(http.user_agent contains "cloudsystemnetwork") or
(http.user_agent contains "cognitiveseo") or
(http.user_agent contains "Collector") or
(http.user_agent contains "cosmos") or
(http.user_agent contains "CrazyWebCrawler") or
(http.user_agent contains "Crescent") or
(http.user_agent contains "Devil") or
((lower(http.user_agent) contains "domain") and ((http.user_agent contains "spider") or (http.user_agent contains "stat") or (http.user_agent contains "Appender") or (http.user_agent contains "Crawler"))) or
(http.user_agent contains "DittoSpyder") or
(http.user_agent contains "Konqueror") or
(http.user_agent contains "Easou") or (http.user_agent contains "Yisou") or (http.user_agent contains "Etao") or
(http.user_agent contains "mail" and ((http.user_agent contains "olf") or (http.user_agent contains "spider"))) or
(http.user_agent contains "exabot.com") or
(http.user_agent contains "getintent") or
(http.user_agent contains "Grabber") or
(http.user_agent contains "GrabNet") or
(http.user_agent contains "HEADMasterSEO") or
(http.user_agent contains "heritrix") or
(http.user_agent contains "htmlparser") or
(http.user_agent contains "hubspot") or
(http.user_agent contains "Jyxobot") or
(lower(http.user_agent) contains "kraken") or
(http.user_agent contains "larbin") or
(http.user_agent contains "ltx71") or
(http.user_agent contains "leiki") or
(http.user_agent contains "LinkScan") or
(http.user_agent contains "Magnet") or
(http.user_agent contains "Mag-Net") or
(http.user_agent contains "Mechanize") or
(http.user_agent contains "MegaIndex") or
(http.user_agent contains "Metasearch") or
(http.user_agent contains "MJ12bot") or
(http.user_agent contains "moz.com") or
(http.user_agent contains "Navroad") or
(http.user_agent contains "Netcraft") or
(http.user_agent contains "niki-bot") or
(http.user_agent contains "NimbleCrawler") or
(http.user_agent contains "Nimbostratus") or
(http.user_agent contains "Ninja") or
(http.user_agent contains "Openfind") or
(http.user_agent contains "Page" and http.user_agent contains "Analyzer") or
(http.user_agent contains "Pixray") or
(http.user_agent contains "probethenet") or
(http.user_agent contains "proximic") or
(http.user_agent contains "psbot") or
(http.user_agent contains "RankActive") or (http.user_agent contains "RankingBot") or (http.user_agent contains "RankurBot") or
(http.user_agent contains "Reaper") or
(http.user_agent contains "SalesIntelligent") or
(http.user_agent contains "Semrush") or
(http.user_agent contains "SEOkicks") or
(http.user_agent contains "spbot") or
(http.user_agent contains "SEOstats") or
(http.user_agent contains "Snapbot") or
(http.user_agent contains "Stripper") or
(http.user_agent contains "Siteimprove") or
(http.user_agent contains "sitesell") or
(http.user_agent contains "Siphon") or
(http.user_agent contains "Sucker") or
(http.user_agent contains "TenFourFox") or
(http.user_agent contains "TurnitinBot") or
(http.user_agent contains "trendiction") or
(http.user_agent contains "twingly") or
(http.user_agent contains "VidibleScraper") or
(http.user_agent contains "WebLeacher") or
(http.user_agent contains "WebmasterWorldForum") or
(http.user_agent contains "webmeup") or
(http.user_agent contains "Webster") or
(http.user_agent contains "Widow") or
(http.user_agent contains "Xaldon") or
(http.user_agent contains "Xenu") or
(http.user_agent contains "xtractor") or
(http.user_agent contains "Zermelo")