1 # Generic exclusion (makes never sense)
9 Disallow: /birthday_confirm.php
11 Disallow: /confirm.php
14 Disallow: /doubler.php
16 Disallow: /install.php
18 Disallow: /lead-confirm.php
21 Disallow: /mailid_top.php
22 Disallow: /network-vcheck.php
23 Disallow: /network-delurl.php
25 Disallow: /register.php
26 Disallow: /show_bonus.php
27 Disallow: /sponsor_confirm.php
28 Disallow: /sponsor_ref.php
29 Disallow: /surfbar.php
37 Disallow: /cms/index/wht/impressum
39 # Exclusion for Google (he will not listen to '*' :( )
45 Disallow: /autoreg.php
47 Disallow: /birthday_confirm.php
49 Disallow: /confirm.php
52 Disallow: /doubler.php
54 Disallow: /install.php
56 Disallow: /lead-confirm.php
59 Disallow: /mailid_top.php
60 Disallow: /network-vcheck.php
61 Disallow: /network-delurl.php
63 Disallow: /register.php
64 Disallow: /show_bonus.php
65 Disallow: /sponsor_confirm.php
66 Disallow: /sponsor_ref.php
67 Disallow: /surfbar.php
75 Disallow: /cms/index/wht/impressum
77 # A lot "bot" you may don't like as I do
78 User-agent: grub-client
99 User-agent: Python-urllib
102 User-agent: NetMechanic
105 User-agent: URL_Spider_Pro
108 User-agent: CherryPicker
111 User-agent: EmailCollector
114 User-agent: EmailSiphon
117 User-agent: WebBandit
120 User-agent: EmailWolf
123 User-agent: ExtractorPro
126 User-agent: CopyRightCheck
132 User-agent: SiteSnagger
135 User-agent: ProWebWalker
138 User-agent: CheeseBot
141 User-agent: LNSpiderguy
147 User-agent: TeleportPro
156 User-agent: Website Quester
159 User-agent: moget/2.1
162 User-agent: WebZip/4.0
165 User-agent: WebStripper
168 User-agent: WebSauger
171 User-agent: WebCopier
177 User-agent: Mister PiX
186 User-agent: WWW-Collector-E
192 User-agent: libWeb/clsHTTP
207 User-agent: InfoNaviRobot
210 User-agent: Harvest/1.5
213 User-agent: Bullseye/1.0
216 User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)
219 User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
222 User-agent: CherryPickerSE/1.0
225 User-agent: CherryPickerElite/1.0
228 User-agent: WebBandit/3.50
231 User-agent: NICErsPRO
234 User-agent: Microsoft URL Control - 5.01.4511
237 User-agent: DittoSpyder
243 User-agent: WebmasterWorldForumBot
252 User-agent: lwp-trivial/1.34
255 User-agent: lwp-trivial
258 User-agent: BunnySlippers
261 User-agent: Microsoft URL Control - 6.00.8169
264 User-agent: URLy Warning
270 User-agent: Wget/1.5.3
276 User-agent: LinkWalker
288 User-agent: humanlinks
291 User-agent: LinkextractorPro
294 User-agent: Offline Explorer
297 User-agent: Mata Hari
303 User-agent: Web Image Collector
306 User-agent: The Intraformant
309 User-agent: True_Robot/1.0
312 User-agent: True_Robot
315 User-agent: BlowFish/1.0
321 User-agent: MIIxpc/4.2
324 User-agent: BuiltBotTough
327 User-agent: ProPowerBot/2.14
330 User-agent: BackDoorBot/1.0
333 User-agent: toCrawl/UrlDispatcher
336 User-agent: WebEnhancer
342 User-agent: VCI WebViewer VCI WebViewer Win32
348 User-agent: Szukacz/1.4
351 User-agent: QueryN Metasearch
354 User-agent: Openfind data gathere
360 User-agent: Xenu's Link Sleuth 1.1c
369 User-agent: RepoMonkey Bait & Tackle/v1.01
372 User-agent: RepoMonkey
375 User-agent: Microsoft URL Control
381 User-agent: URL Control
384 User-agent: Zeus Link Scout
387 User-agent: Zeus 32297 Webster Pro V2.9 Win32
390 User-agent: Webster Pro
393 User-agent: EroCrawler
396 User-agent: LinkScan/8.1a Unix
399 User-agent: Keyword Density/0.9
402 User-agent: Kenjin Spider
405 User-agent: Iron33/1.0.2
408 User-agent: Bookmark search tool
411 User-agent: GetRight/4.2
414 User-agent: FairAd Client
420 User-agent: Aqua_Products
423 User-agent: Radiation Retriever 1.1
426 User-agent: Flaming AttackBot
429 User-agent: Oracle Ultra Search
432 User-agent: MSIECrawler
438 User-agent: searchpreview
441 # This is for my web pages using spider-trap
443 Disallow: /guestbook/
444 Disallow: /blacklist.txt
445 Disallow: /whitelist.txt
447 User-agent: Googlebot
448 Disallow: /guestbook/
449 Disallow: /blacklist.txt
450 Disallow: /whitelist.txt