1 # Generic exclusion (makes never sense)
9 Disallow: /birthday_confirm.php
11 Disallow: /confirm.php
14 Disallow: /doubler.php
16 Disallow: /install.php
18 Disallow: /lead-confirm.php
21 Disallow: /mailid_top.php
22 Disallow: /network-vcheck.php
23 Disallow: /network-delurl.php
25 Disallow: /show_bonus.php
26 Disallow: /sponsor_confirm.php
27 Disallow: /sponsor_ref.php
28 Disallow: /surfbar.php
34 Disallow: /cms/index/wht/impressum
36 # Exclusion for Google (he will not listen to '*' :( )
42 Disallow: /autoreg.php
44 Disallow: /birthday_confirm.php
46 Disallow: /confirm.php
49 Disallow: /doubler.php
51 Disallow: /install.php
53 Disallow: /lead-confirm.php
56 Disallow: /mailid_top.php
57 Disallow: /network-vcheck.php
58 Disallow: /network-delurl.php
60 Disallow: /show_bonus.php
61 Disallow: /sponsor_confirm.php
62 Disallow: /sponsor_ref.php
63 Disallow: /surfbar.php
69 Disallow: /cms/index/wht/impressum
71 # A lot "bot" you may don't like as I do
72 User-agent: grub-client
93 User-agent: Python-urllib
96 User-agent: NetMechanic
99 User-agent: URL_Spider_Pro
102 User-agent: CherryPicker
105 User-agent: EmailCollector
108 User-agent: EmailSiphon
111 User-agent: WebBandit
114 User-agent: EmailWolf
117 User-agent: ExtractorPro
120 User-agent: CopyRightCheck
126 User-agent: SiteSnagger
129 User-agent: ProWebWalker
132 User-agent: CheeseBot
135 User-agent: LNSpiderguy
141 User-agent: TeleportPro
150 User-agent: Website Quester
153 User-agent: moget/2.1
156 User-agent: WebZip/4.0
159 User-agent: WebStripper
162 User-agent: WebSauger
165 User-agent: WebCopier
171 User-agent: Mister PiX
180 User-agent: WWW-Collector-E
186 User-agent: libWeb/clsHTTP
201 User-agent: InfoNaviRobot
204 User-agent: Harvest/1.5
207 User-agent: Bullseye/1.0
210 User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)
213 User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
216 User-agent: CherryPickerSE/1.0
219 User-agent: CherryPickerElite/1.0
222 User-agent: WebBandit/3.50
225 User-agent: NICErsPRO
228 User-agent: Microsoft URL Control - 5.01.4511
231 User-agent: DittoSpyder
237 User-agent: WebmasterWorldForumBot
246 User-agent: lwp-trivial/1.34
249 User-agent: lwp-trivial
252 User-agent: BunnySlippers
255 User-agent: Microsoft URL Control - 6.00.8169
258 User-agent: URLy Warning
264 User-agent: Wget/1.5.3
270 User-agent: LinkWalker
282 User-agent: humanlinks
285 User-agent: LinkextractorPro
288 User-agent: Offline Explorer
291 User-agent: Mata Hari
297 User-agent: Web Image Collector
300 User-agent: The Intraformant
303 User-agent: True_Robot/1.0
306 User-agent: True_Robot
309 User-agent: BlowFish/1.0
315 User-agent: MIIxpc/4.2
318 User-agent: BuiltBotTough
321 User-agent: ProPowerBot/2.14
324 User-agent: BackDoorBot/1.0
327 User-agent: toCrawl/UrlDispatcher
330 User-agent: WebEnhancer
336 User-agent: VCI WebViewer VCI WebViewer Win32
342 User-agent: Szukacz/1.4
345 User-agent: QueryN Metasearch
348 User-agent: Openfind data gathere
354 User-agent: Xenu's Link Sleuth 1.1c
363 User-agent: RepoMonkey Bait & Tackle/v1.01
366 User-agent: RepoMonkey
369 User-agent: Microsoft URL Control
375 User-agent: URL Control
378 User-agent: Zeus Link Scout
381 User-agent: Zeus 32297 Webster Pro V2.9 Win32
384 User-agent: Webster Pro
387 User-agent: EroCrawler
390 User-agent: LinkScan/8.1a Unix
393 User-agent: Keyword Density/0.9
396 User-agent: Kenjin Spider
399 User-agent: Iron33/1.0.2
402 User-agent: Bookmark search tool
405 User-agent: GetRight/4.2
408 User-agent: FairAd Client
414 User-agent: Aqua_Products
417 User-agent: Radiation Retriever 1.1
420 User-agent: Flaming AttackBot
423 User-agent: Oracle Ultra Search
426 User-agent: MSIECrawler
432 User-agent: searchpreview
435 # This is for my web pages using spider-trap
437 Disallow: /guestbook/
438 Disallow: /blacklist.txt
439 Disallow: /whitelist.txt
441 User-agent: Googlebot
442 Disallow: /guestbook/
443 Disallow: /blacklist.txt
444 Disallow: /whitelist.txt