Commit 2cf618e1 authored by omega8cc's avatar omega8cc

Add Nutch to the list of blocked crawlers in Nginx.

parent db25410a
......@@ -151,8 +151,8 @@ map $request_uri $key_uri {
### Deny crawlers without 403 response.
###
map $http_user_agent $is_crawler {
default '';
~*HTTrack|MJ12bot|HTMLParser|libwww|PECL|AutomaticSiteMap|ClickSense|ValueClick|SiteBot|BuzzTracker|sistrix|Offline|Screaming is_crawler;
default '';
~*HTTrack|MJ12bot|HTMLParser|libwww|PECL|AutomaticSiteMap|ClickSense|ValueClick|SiteBot|BuzzTracker|sistrix|Offline|Screaming|Nutch is_crawler;
}
###
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment