1 # Generic exclusion (makes never sense)
9 Disallow: /birthday_confirm.php
11 Disallow: /confirm.php
14 Disallow: /doubler.php
16 Disallow: /install.php
18 Disallow: /lead-confirm.php
21 Disallow: /mailid_top.php
22 Disallow: /network-vcheck.php
23 Disallow: /network-delurl.php
25 Disallow: /show_bonus.php
26 Disallow: /sponsor_confirm.php
27 Disallow: /sponsor_ref.php
28 Disallow: /surfbar.php
36 Disallow: /cms/index/wht/impressum
38 # Exclusion for Google (he will not listen to '*' :( )
44 Disallow: /autoreg.php
46 Disallow: /birthday_confirm.php
48 Disallow: /confirm.php
51 Disallow: /doubler.php
53 Disallow: /install.php
55 Disallow: /lead-confirm.php
58 Disallow: /mailid_top.php
59 Disallow: /network-vcheck.php
60 Disallow: /network-delurl.php
62 Disallow: /show_bonus.php
63 Disallow: /sponsor_confirm.php
64 Disallow: /sponsor_ref.php
65 Disallow: /surfbar.php
73 Disallow: /cms/index/wht/impressum
75 # A lot "bot" you may don't like as I do
76 User-agent: grub-client
97 User-agent: Python-urllib
100 User-agent: NetMechanic
103 User-agent: URL_Spider_Pro
106 User-agent: CherryPicker
109 User-agent: EmailCollector
112 User-agent: EmailSiphon
115 User-agent: WebBandit
118 User-agent: EmailWolf
121 User-agent: ExtractorPro
124 User-agent: CopyRightCheck
130 User-agent: SiteSnagger
133 User-agent: ProWebWalker
136 User-agent: CheeseBot
139 User-agent: LNSpiderguy
145 User-agent: TeleportPro
154 User-agent: Website Quester
157 User-agent: moget/2.1
160 User-agent: WebZip/4.0
163 User-agent: WebStripper
166 User-agent: WebSauger
169 User-agent: WebCopier
175 User-agent: Mister PiX
184 User-agent: WWW-Collector-E
190 User-agent: libWeb/clsHTTP
205 User-agent: InfoNaviRobot
208 User-agent: Harvest/1.5
211 User-agent: Bullseye/1.0
214 User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)
217 User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
220 User-agent: CherryPickerSE/1.0
223 User-agent: CherryPickerElite/1.0
226 User-agent: WebBandit/3.50
229 User-agent: NICErsPRO
232 User-agent: Microsoft URL Control - 5.01.4511
235 User-agent: DittoSpyder
241 User-agent: WebmasterWorldForumBot
250 User-agent: lwp-trivial/1.34
253 User-agent: lwp-trivial
256 User-agent: BunnySlippers
259 User-agent: Microsoft URL Control - 6.00.8169
262 User-agent: URLy Warning
268 User-agent: Wget/1.5.3
274 User-agent: LinkWalker
286 User-agent: humanlinks
289 User-agent: LinkextractorPro
292 User-agent: Offline Explorer
295 User-agent: Mata Hari
301 User-agent: Web Image Collector
304 User-agent: The Intraformant
307 User-agent: True_Robot/1.0
310 User-agent: True_Robot
313 User-agent: BlowFish/1.0
319 User-agent: MIIxpc/4.2
322 User-agent: BuiltBotTough
325 User-agent: ProPowerBot/2.14
328 User-agent: BackDoorBot/1.0
331 User-agent: toCrawl/UrlDispatcher
334 User-agent: WebEnhancer
340 User-agent: VCI WebViewer VCI WebViewer Win32
346 User-agent: Szukacz/1.4
349 User-agent: QueryN Metasearch
352 User-agent: Openfind data gathere
358 User-agent: Xenu's Link Sleuth 1.1c
367 User-agent: RepoMonkey Bait & Tackle/v1.01
370 User-agent: RepoMonkey
373 User-agent: Microsoft URL Control
379 User-agent: URL Control
382 User-agent: Zeus Link Scout
385 User-agent: Zeus 32297 Webster Pro V2.9 Win32
388 User-agent: Webster Pro
391 User-agent: EroCrawler
394 User-agent: LinkScan/8.1a Unix
397 User-agent: Keyword Density/0.9
400 User-agent: Kenjin Spider
403 User-agent: Iron33/1.0.2
406 User-agent: Bookmark search tool
409 User-agent: GetRight/4.2
412 User-agent: FairAd Client
418 User-agent: Aqua_Products
421 User-agent: Radiation Retriever 1.1
424 User-agent: Flaming AttackBot
427 User-agent: Oracle Ultra Search
430 User-agent: MSIECrawler
436 User-agent: searchpreview
439 # This is for my web pages using spider-trap
441 Disallow: /guestbook/
442 Disallow: /blacklist.txt
443 Disallow: /whitelist.txt
445 User-agent: Googlebot
446 Disallow: /guestbook/
447 Disallow: /blacklist.txt
448 Disallow: /whitelist.txt