1 # Generic exclusion (makes never sense)
9 Disallow: /birthday_confirm.php
11 Disallow: /confirm.php
14 Disallow: /doubler.php
16 Disallow: /install.php
18 Disallow: /lead-confirm.php
21 Disallow: /mailid_top.php
22 Disallow: /network-vcheck.php
24 Disallow: /show_bonus.php
25 Disallow: /sponsor_confirm.php
26 Disallow: /sponsor_ref.php
27 Disallow: /surfbar.php
33 Disallow: /cms/index/wht/impressum
35 # Exclusion for Google (he will not listen to '*' :( )
41 Disallow: /autoreg.php
43 Disallow: /birthday_confirm.php
45 Disallow: /confirm.php
48 Disallow: /doubler.php
50 Disallow: /install.php
52 Disallow: /lead-confirm.php
55 Disallow: /mailid_top.php
56 Disallow: /network-vcheck.php
58 Disallow: /show_bonus.php
59 Disallow: /sponsor_confirm.php
60 Disallow: /sponsor_ref.php
61 Disallow: /surfbar.php
67 Disallow: /cms/index/wht/impressum
69 # A lot "bot" you may don't like as I do
70 User-agent: grub-client
91 User-agent: Python-urllib
94 User-agent: NetMechanic
97 User-agent: URL_Spider_Pro
100 User-agent: CherryPicker
103 User-agent: EmailCollector
106 User-agent: EmailSiphon
109 User-agent: WebBandit
112 User-agent: EmailWolf
115 User-agent: ExtractorPro
118 User-agent: CopyRightCheck
124 User-agent: SiteSnagger
127 User-agent: ProWebWalker
130 User-agent: CheeseBot
133 User-agent: LNSpiderguy
139 User-agent: TeleportPro
148 User-agent: Website Quester
151 User-agent: moget/2.1
154 User-agent: WebZip/4.0
157 User-agent: WebStripper
160 User-agent: WebSauger
163 User-agent: WebCopier
169 User-agent: Mister PiX
178 User-agent: WWW-Collector-E
184 User-agent: libWeb/clsHTTP
199 User-agent: InfoNaviRobot
202 User-agent: Harvest/1.5
205 User-agent: Bullseye/1.0
208 User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)
211 User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
214 User-agent: CherryPickerSE/1.0
217 User-agent: CherryPickerElite/1.0
220 User-agent: WebBandit/3.50
223 User-agent: NICErsPRO
226 User-agent: Microsoft URL Control - 5.01.4511
229 User-agent: DittoSpyder
235 User-agent: WebmasterWorldForumBot
244 User-agent: lwp-trivial/1.34
247 User-agent: lwp-trivial
250 User-agent: BunnySlippers
253 User-agent: Microsoft URL Control - 6.00.8169
256 User-agent: URLy Warning
262 User-agent: Wget/1.5.3
268 User-agent: LinkWalker
280 User-agent: humanlinks
283 User-agent: LinkextractorPro
286 User-agent: Offline Explorer
289 User-agent: Mata Hari
295 User-agent: Web Image Collector
298 User-agent: The Intraformant
301 User-agent: True_Robot/1.0
304 User-agent: True_Robot
307 User-agent: BlowFish/1.0
313 User-agent: MIIxpc/4.2
316 User-agent: BuiltBotTough
319 User-agent: ProPowerBot/2.14
322 User-agent: BackDoorBot/1.0
325 User-agent: toCrawl/UrlDispatcher
328 User-agent: WebEnhancer
334 User-agent: VCI WebViewer VCI WebViewer Win32
340 User-agent: Szukacz/1.4
343 User-agent: QueryN Metasearch
346 User-agent: Openfind data gathere
352 User-agent: Xenu's Link Sleuth 1.1c
361 User-agent: RepoMonkey Bait & Tackle/v1.01
364 User-agent: RepoMonkey
367 User-agent: Microsoft URL Control
373 User-agent: URL Control
376 User-agent: Zeus Link Scout
379 User-agent: Zeus 32297 Webster Pro V2.9 Win32
382 User-agent: Webster Pro
385 User-agent: EroCrawler
388 User-agent: LinkScan/8.1a Unix
391 User-agent: Keyword Density/0.9
394 User-agent: Kenjin Spider
397 User-agent: Iron33/1.0.2
400 User-agent: Bookmark search tool
403 User-agent: GetRight/4.2
406 User-agent: FairAd Client
412 User-agent: Aqua_Products
415 User-agent: Radiation Retriever 1.1
418 User-agent: Flaming AttackBot
421 User-agent: Oracle Ultra Search
424 User-agent: MSIECrawler
430 User-agent: searchpreview
433 # This is for my web pages using spider-trap
435 Disallow: /guestbook/
436 Disallow: /blacklist.txt
437 Disallow: /whitelist.txt
439 User-agent: Googlebot
440 Disallow: /guestbook/
441 Disallow: /blacklist.txt
442 Disallow: /whitelist.txt