1 # Generic exclusion (makes never sense)
9 Disallow: /birthday_confirm.php
11 Disallow: /confirm.php
14 Disallow: /doubler.php
16 Disallow: /install.php
18 Disallow: /lead-confirm.php
21 Disallow: /mailid_top.php
22 Disallow: /network-vcheck.php
23 Disallow: /network-delurl.php
25 Disallow: /show_bonus.php
26 Disallow: /sponsor_confirm.php
27 Disallow: /sponsor_ref.php
28 Disallow: /surfbar.php
36 Disallow: /cms/index/wht/impressum
38 # Exclusion for Google (he will not listen to '*' :( )
44 Disallow: /autoreg.php
46 Disallow: /birthday_confirm.php
48 Disallow: /confirm.php
51 Disallow: /doubler.php
53 Disallow: /install.php
55 Disallow: /lead-confirm.php
58 Disallow: /mailid_top.php
59 Disallow: /network-vcheck.php
60 Disallow: /network-delurl.php
62 Disallow: /show_bonus.php
63 Disallow: /sponsor_confirm.php
64 Disallow: /sponsor_ref.php
65 Disallow: /surfbar.php
73 Disallow: /cms/index/wht/impressum
75 # A lot "bot" you may don't like as I do
76 User-agent: grub-client
97 User-agent: Python-urllib
100 User-agent: NetMechanic
103 User-agent: URL_Spider_Pro
106 User-agent: CherryPicker
109 User-agent: EmailCollector
112 User-agent: EmailSiphon
115 User-agent: WebBandit
118 User-agent: EmailWolf
121 User-agent: ExtractorPro
124 User-agent: CopyRightCheck
130 User-agent: SiteSnagger
133 User-agent: ProWebWalker
136 User-agent: CheeseBot
139 User-agent: LNSpiderguy
145 User-agent: TeleportPro
154 User-agent: Website Quester
157 User-agent: moget/2.1
160 User-agent: WebZip/4.0
163 User-agent: WebStripper
166 User-agent: WebSauger
169 User-agent: WebCopier
175 User-agent: Mister PiX
184 User-agent: WWW-Collector-E
190 User-agent: libWeb/clsHTTP
205 User-agent: InfoNaviRobot
208 User-agent: Harvest/1.5
211 User-agent: Bullseye/1.0
214 User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)
217 User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
220 User-agent: CherryPickerSE/1.0
223 User-agent: CherryPickerElite/1.0
226 User-agent: WebBandit/3.50
229 User-agent: NICErsPRO
232 User-agent: Microsoft URL Control - 5.01.4511
235 User-agent: DittoSpyder
241 User-agent: WebmasterWorldForumBot
250 User-agent: lwp-trivial/1.34
253 User-agent: lwp-trivial
256 User-agent: BunnySlippers
259 User-agent: Microsoft URL Control - 6.00.8169
262 User-agent: URLy Warning
265 User-agent: LinkWalker
277 User-agent: humanlinks
280 User-agent: LinkextractorPro
283 User-agent: Offline Explorer
286 User-agent: Mata Hari
292 User-agent: Web Image Collector
295 User-agent: The Intraformant
298 User-agent: True_Robot/1.0
301 User-agent: True_Robot
304 User-agent: BlowFish/1.0
310 User-agent: MIIxpc/4.2
313 User-agent: BuiltBotTough
316 User-agent: ProPowerBot/2.14
319 User-agent: BackDoorBot/1.0
322 User-agent: toCrawl/UrlDispatcher
325 User-agent: WebEnhancer
331 User-agent: VCI WebViewer VCI WebViewer Win32
337 User-agent: Szukacz/1.4
340 User-agent: QueryN Metasearch
343 User-agent: Openfind data gathere
349 User-agent: Xenu's Link Sleuth 1.1c
358 User-agent: RepoMonkey Bait & Tackle/v1.01
361 User-agent: RepoMonkey
364 User-agent: Microsoft URL Control
370 User-agent: URL Control
373 User-agent: Zeus Link Scout
376 User-agent: Zeus 32297 Webster Pro V2.9 Win32
379 User-agent: Webster Pro
382 User-agent: EroCrawler
385 User-agent: LinkScan/8.1a Unix
388 User-agent: Keyword Density/0.9
391 User-agent: Kenjin Spider
394 User-agent: Iron33/1.0.2
397 User-agent: Bookmark search tool
400 User-agent: GetRight/4.2
403 User-agent: FairAd Client
409 User-agent: Aqua_Products
412 User-agent: Radiation Retriever 1.1
415 User-agent: Flaming AttackBot
418 User-agent: Oracle Ultra Search
421 User-agent: MSIECrawler
427 User-agent: searchpreview
430 # This is for my web pages using spider-trap
432 Disallow: /guestbook/
433 Disallow: /blacklist.txt
434 Disallow: /whitelist.txt
436 User-agent: Googlebot
437 Disallow: /guestbook/
438 Disallow: /blacklist.txt
439 Disallow: /whitelist.txt