X-Git-Url: https://git.mxchange.org/?p=mailer.git;a=blobdiff_plain;f=robots.txt;h=4d1db761fa2fbb6447001ed9a650f87a7e62c5a0;hp=5d2c9ad204ac2f9996faf2f62294fef8cdb862db;hb=638e7fa034b3281e356a6e36b1380571e8860115;hpb=64c8349613addc3da2242c5cd6b99d64e3fb5f8e diff --git a/robots.txt b/robots.txt index 5d2c9ad204..4d1db761fa 100644 --- a/robots.txt +++ b/robots.txt @@ -83,12 +83,6 @@ Disallow: / User-agent: LNSpiderguy Disallow: / -User-agent: ia_archiver -Disallow: / - -User-agent: ia_archiver/1.6 -Disallow: / - User-agent: Teleport Disallow: /