X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=robots.txt;h=4d1db761fa2fbb6447001ed9a650f87a7e62c5a0;hb=8fa01ac88b1208b28b47f57d997db6223f60cb9c;hp=6e16af6709b85b893ba0d8d8a67ee1aaf7847f34;hpb=75ad748a68473ace540251427a74fb781b1145e9;p=mailer.git

diff --git a/robots.txt b/robots.txt
index 6e16af6709..4d1db761fa 100644
--- a/robots.txt
+++ b/robots.txt
@@ -5,13 +5,16 @@ User-agent: *
 Disallow: /css/
 Disallow: /inc/
 Disallow: /images/
-Disallow: /sql-upgrades/
 Disallow: /templates/
-Disallow: /upgrade.php
 Disallow: /install.php
 Disallow: /cms/index/wht/impressum
 
-User-agent: *
+User-agent: Googlebot
+Disallow: /css/
+Disallow: /inc/
+Disallow: /images/
+Disallow: /templates/
+Disallow: /install.php
 Disallow: /cms/index/wht/impressum
 
 User-agent: grub-client
@@ -80,12 +83,6 @@ Disallow: /
 User-agent: LNSpiderguy
 Disallow: /
 
-User-agent: ia_archiver
-Disallow: /
-
-User-agent: ia_archiver/1.6
-Disallow: /
-
 User-agent: Teleport
 Disallow: /
 
@@ -383,20 +380,13 @@ Disallow: /
 User-agent: searchpreview
 Disallow: /
 
+# This is for my webpages
 User-agent: *
-Disallow: /guestbook
-
-User-agent: Googlebot
-Disallow: /guestbook
-
-User-agent: *
-Disallow: /blacklist.txt
-
-User-agent: Googlebot
+Disallow: /guestbook/
 Disallow: /blacklist.txt
-
-User-agent: *
 Disallow: /whitelist.txt
 
 User-agent: Googlebot
+Disallow: /guestbook/
+Disallow: /blacklist.txt
 Disallow: /whitelist.txt