also prevent it in .htacces. You may want to add this to one of your files in /etc...

[mailer.git] / robots.txt

diff --git a/robots.txt b/robots.txt
index deb99838938f81e30ac406dadeedb252e4596571..e9fc0fd59c66f6177fa1afd94be4ce6324b8cf81 100644 (file)
--- a/robots.txt
+++ b/robots.txt
@@ -20,6 +20,7 @@ Disallow: /login.php
 Disallow: /mailid.php
 Disallow: /mailid_top.php
 Disallow: /network-vcheck.php
+Disallow: /network-delurl.php
 Disallow: /ref.php
 Disallow: /show_bonus.php
 Disallow: /sponsor_confirm.php
@@ -54,6 +55,7 @@ Disallow: /login.php
 Disallow: /mailid.php
 Disallow: /mailid_top.php
 Disallow: /network-vcheck.php
+Disallow: /network-delurl.php
 Disallow: /ref.php
 Disallow: /show_bonus.php
 Disallow: /sponsor_confirm.php
@@ -430,7 +432,7 @@ Disallow: /
 User-agent: searchpreview
 Disallow: /
 
-# This is for my webpages
+# This is for my web pages using spider-trap
 User-agent: *
 Disallow: /guestbook/
 Disallow: /blacklist.txt