Hotfix

[mailer.git] / robots.txt

diff --git a/robots.txt b/robots.txt
index 5d2c9ad204ac2f9996faf2f62294fef8cdb862db..57a10727bf0d042c10b2209ce318ccfe768e09b5 100644 (file)
--- a/robots.txt
+++ b/robots.txt
@@ -83,12 +83,6 @@ Disallow: /
 User-agent: LNSpiderguy
 Disallow: /
 
-User-agent: ia_archiver
-Disallow: /
-
-User-agent: ia_archiver/1.6
-Disallow: /
-
 User-agent: Teleport
 Disallow: /
 
@@ -212,15 +206,6 @@ Disallow: /
 User-agent: URLy Warning
 Disallow: /
 
-User-agent: Wget/1.6
-Disallow: /
-
-User-agent: Wget/1.5.3
-Disallow: /
-
-User-agent: Wget
-Disallow: /
-
 User-agent: LinkWalker
 Disallow: /
 
@@ -396,3 +381,6 @@ User-agent: Googlebot
 Disallow: /guestbook/
 Disallow: /blacklist.txt
 Disallow: /whitelist.txt
+
+User-agent: JobRoboter
+Disallow: /