diff MoinMoin/config/multiconfig.py @ 3012:ca0bf3aefa0f

added more user agents to the spider list (port from 1.6)
author Thomas Waldmann <tw AT waldmann-edv DOT de>
date Sun, 06 Jan 2008 19:20:55 +0100
parents 0ae378dc1edf
children f0250b5a6982
line wrap: on
line diff
--- a/MoinMoin/config/multiconfig.py	Sun Jan 06 19:18:19 2008 +0100
+++ b/MoinMoin/config/multiconfig.py	Sun Jan 06 19:20:55 2008 +0100
@@ -599,9 +599,10 @@
 
     # a regex of HTTP_USER_AGENTS that should be excluded from logging
     # and receive a FORBIDDEN for anything except viewing a page
-    ua_spiders = ('archiver|cfetch|crawler|curl|gigabot|googlebot|holmes|htdig|httrack|httpunit|jeeves|larbin|leech|'
-                  'linkbot|linkmap|linkwalk|mercator|mirror|msnbot|msrbot|neomo|nutbot|omniexplorer|puf|robot|scooter|seekbot|'
-                  'sherlock|slurp|sitecheck|spider|teleport|voyager|webreaper|wget')
+    ua_spiders = ('archiver|cfetch|charlotte|crawler|curl|gigabot|googlebot|heritrix|holmes|htdig|httrack|httpunit|'
+                  'intelix|java|jeeves|larbin|leech|libwww-perl|linkbot|linkmap|linkwalk|litefinder|mercator|'
+                  'microsoft.url.control|mirror| mj12bot|msnbot|msrbot|neomo|nutbot|omniexplorer|puf|robot|scooter|seekbot|'
+                  'sherlock|slurp|sitecheck|snoopy|spider|teleport|twiceler|voilabot|voyager|webreaper|wget|yeti')
 
     # Wiki identity
     sitename = u'Untitled Wiki'