X-Git-Url: https://git.mxchange.org/?p=mailer.git;a=blobdiff_plain;f=robots.txt;h=e9fc0fd59c66f6177fa1afd94be4ce6324b8cf81;hp=7b26c7d62ea6f5cfa2179b1e6e2b7583d6c74880;hb=c4be539d249046d6d64d2181c3a1e898fe9a64f5;hpb=e52dff4c86ee53e5aa59f2ee0f5345f1731c9623

diff --git a/robots.txt b/robots.txt
index 7b26c7d62e..e9fc0fd59c 100644
--- a/robots.txt
+++ b/robots.txt
@@ -1,19 +1,74 @@
-#
-# Thanx to Mr. Jungwirth for his hint to create this file (was a bit lost in my mind...)
-#
+# Generic exclusion (makes never sense)
 User-agent: *
+Disallow: /admin.php
+Disallow: /agb.php
+Disallow: /ajax.php
+Disallow: /api.php
+Disallow: /autoreg.php
+Disallow: /beg.php
+Disallow: /birthday_confirm.php
+Disallow: /click.php
+Disallow: /confirm.php
+Disallow: /css.php
+Disallow: /debug.php
+Disallow: /doubler.php
+Disallow: /img.php
+Disallow: /install.php
+Disallow: /js.php
+Disallow: /lead-confirm.php
+Disallow: /login.php
+Disallow: /mailid.php
+Disallow: /mailid_top.php
+Disallow: /network-vcheck.php
+Disallow: /network-delurl.php
+Disallow: /ref.php
+Disallow: /show_bonus.php
+Disallow: /sponsor_confirm.php
+Disallow: /sponsor_ref.php
+Disallow: /surfbar.php
+Disallow: /view.php
 Disallow: /css/
 Disallow: /inc/
 Disallow: /images/
-Disallow: /sql-upgrades/
 Disallow: /templates/
-Disallow: /upgrade.php
-Disallow: /install.php
 Disallow: /cms/index/wht/impressum
 
-User-agent: *
+# Exclusion for Google (he will not listen to '*' :( )
+User-agent: Googlebot
+Disallow: /admin.php
+Disallow: /agb.php
+Disallow: /ajax.php
+Disallow: /api.php
+Disallow: /autoreg.php
+Disallow: /beg.php
+Disallow: /birthday_confirm.php
+Disallow: /click.php
+Disallow: /confirm.php
+Disallow: /css.php
+Disallow: /debug.php
+Disallow: /doubler.php
+Disallow: /img.php
+Disallow: /install.php
+Disallow: /js.php
+Disallow: /lead-confirm.php
+Disallow: /login.php
+Disallow: /mailid.php
+Disallow: /mailid_top.php
+Disallow: /network-vcheck.php
+Disallow: /network-delurl.php
+Disallow: /ref.php
+Disallow: /show_bonus.php
+Disallow: /sponsor_confirm.php
+Disallow: /sponsor_ref.php
+Disallow: /surfbar.php
+Disallow: /view.php
+Disallow: /css/
+Disallow: /inc/
+Disallow: /images/
+Disallow: /templates/
 Disallow: /cms/index/wht/impressum
 
+# A lot "bot" you may don't like as I do
 User-agent: grub-client
 Disallow: /
 
@@ -80,12 +135,6 @@ Disallow: /
 User-agent: LNSpiderguy
 Disallow: /
 
-User-agent: ia_archiver
-Disallow: /
-
-User-agent: ia_archiver/1.6
-Disallow: /
-
 User-agent: Teleport
 Disallow: /
 
@@ -383,28 +432,13 @@ Disallow: /
 User-agent: searchpreview
 Disallow: /
 
+# This is for my web pages using spider-trap
 User-agent: *
-Disallow: /guestbook
-
-User-agent: Googlebot
-Disallow: /guestbook
-
-User-agent: *
-Disallow: /blacklist.txt
-
-User-agent: Googlebot
+Disallow: /guestbook/
 Disallow: /blacklist.txt
-
-User-agent: *
 Disallow: /whitelist.txt
 
 User-agent: Googlebot
+Disallow: /guestbook/
+Disallow: /blacklist.txt
 Disallow: /whitelist.txt
-
-User-agent: *
-Disallow: /inc/
-Disallow: /templates/
-
-User-agent: Googlebot
-Disallow: /inc/
-Disallow: /templates/