Commit 96e776e1 authored by omega8cc's avatar omega8cc

Nginx: Update crawlers regex

parent 1725f84d
......@@ -250,9 +250,10 @@ map $request_uri $key_uri {
###
map $http_user_agent $is_crawler {
default '';
~*HTTrack|BrokenLinkCheck|2009042316.*Firefox.*3\.0\.10|MJ12|HTMLParser is_crawler;
~*SiteBot|PECL|Automatic|CCBot|BuzzTrack|Sistrix|Offline|Nutch|Mireo is_crawler;
~*SWEB|Morfeus|GSLFbot|HiScan|Riddler|DBot|SEOkicks|PChomebot is_crawler;
~*HTTrack|BrokenLinkCheck|2009042316.*Firefox.*3\.0\.10 is_crawler;
~*SiteBot|PECL|Automatic|CCBot|BuzzTrack|Sistrix|Offline is_crawler;
~*SWEB|Morfeus|GSLFbot|HiScan|Riddler|DBot|SEOkicks|MJ12 is_crawler;
~*PChomebot|Scrap|HTMLParser|Nutch|Mireo|Semrush|Ahrefs is_crawler;
}
###
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment