Added more wrapper, commented out another noisy debug line

[mailer.git] / robots.txt

diff --git a/robots.txt b/robots.txt
index 57a10727bf0d042c10b2209ce318ccfe768e09b5..4306fdb907d89fd909288fb48dbf156caff2e413 100644 (file)
--- a/robots.txt
+++ b/robots.txt
@@ -1,6 +1,4 @@
-#
-# Thanx to Mr. Jungwirth for his hint to create this file (was a bit lost in my mind...)
-#
+# Generic exclusion (makes never sense)
 User-agent: *
 Disallow: /css/
 Disallow: /inc/
@@ -9,6 +7,7 @@ Disallow: /templates/
 Disallow: /install.php
 Disallow: /cms/index/wht/impressum
 
+# Exclusion for Google (he will not listen to '*' :( )
 User-agent: Googlebot
 Disallow: /css/
 Disallow: /inc/
@@ -17,6 +16,7 @@ Disallow: /templates/
 Disallow: /install.php
 Disallow: /cms/index/wht/impressum
 
+# A lot "bot" you may don't like as I do
 User-agent: grub-client
 Disallow: /
 
@@ -206,6 +206,15 @@ Disallow: /
 User-agent: URLy Warning
 Disallow: /
 
+User-agent: Wget/1.6
+Disallow: /
+
+User-agent: Wget/1.5.3
+Disallow: /
+
+User-agent: Wget
+Disallow: /
+
 User-agent: LinkWalker
 Disallow: /
 
@@ -381,6 +390,3 @@ User-agent: Googlebot
 Disallow: /guestbook/
 Disallow: /blacklist.txt
 Disallow: /whitelist.txt
-
-User-agent: JobRoboter
-Disallow: /