No more PHP4:
[mailer.git] / robots.txt
1 # Generic exclusion (makes never sense)
2 User-agent: *
3 Disallow: /admin.php
4 Disallow: /agb.php
5 Disallow: /ajax.php
6 Disallow: /api.php
7 Disallow: /autoreg.php
8 Disallow: /beg.php
9 Disallow: /birthday_confirm.php
10 Disallow: /click.php
11 Disallow: /confirm.php
12 Disallow: /css.php
13 Disallow: /debug.php
14 Disallow: /doubler.php
15 Disallow: /img.php
16 Disallow: /install.php
17 Disallow: /js.php
18 Disallow: /lead-confirm.php
19 Disallow: /login.php
20 Disallow: /mailid.php
21 Disallow: /mailid_top.php
22 Disallow: /network-vcheck.php
23 Disallow: /network-delurl.php
24 Disallow: /ref.php
25 Disallow: /show_bonus.php
26 Disallow: /sponsor_confirm.php
27 Disallow: /sponsor_ref.php
28 Disallow: /surfbar.php
29 Disallow: /view.php
30 Disallow: /assets/
31 Disallow: /css/
32 Disallow: /js/
33 Disallow: /inc/
34 Disallow: /images/
35 Disallow: /templates/
36 Disallow: /cms/index/wht/impressum
37
38 # Exclusion for Google (he will not listen to '*' :( )
39 User-agent: Googlebot
40 Disallow: /admin.php
41 Disallow: /agb.php
42 Disallow: /ajax.php
43 Disallow: /api.php
44 Disallow: /autoreg.php
45 Disallow: /beg.php
46 Disallow: /birthday_confirm.php
47 Disallow: /click.php
48 Disallow: /confirm.php
49 Disallow: /css.php
50 Disallow: /debug.php
51 Disallow: /doubler.php
52 Disallow: /img.php
53 Disallow: /install.php
54 Disallow: /js.php
55 Disallow: /lead-confirm.php
56 Disallow: /login.php
57 Disallow: /mailid.php
58 Disallow: /mailid_top.php
59 Disallow: /network-vcheck.php
60 Disallow: /network-delurl.php
61 Disallow: /ref.php
62 Disallow: /show_bonus.php
63 Disallow: /sponsor_confirm.php
64 Disallow: /sponsor_ref.php
65 Disallow: /surfbar.php
66 Disallow: /view.php
67 Disallow: /assets/
68 Disallow: /css/
69 Disallow: /js/
70 Disallow: /inc/
71 Disallow: /images/
72 Disallow: /templates/
73 Disallow: /cms/index/wht/impressum
74
75 # A lot "bot" you may don't like as I do
76 User-agent: grub-client
77 Disallow: /
78
79 User-agent: grub
80 Disallow: /
81
82 User-agent: looksmart
83 Disallow: /
84
85 User-agent: WebZip
86 Disallow: /
87
88 User-agent: larbin
89 Disallow: /
90
91 User-agent: b2w/0.1
92 Disallow: /
93
94 User-agent: psbot
95 Disallow: /
96
97 User-agent: Python-urllib
98 Disallow: /
99
100 User-agent: NetMechanic
101 Disallow: /
102
103 User-agent: URL_Spider_Pro
104 Disallow: /
105
106 User-agent: CherryPicker
107 Disallow: /
108
109 User-agent: EmailCollector
110 Disallow: /
111
112 User-agent: EmailSiphon
113 Disallow: /
114
115 User-agent: WebBandit
116 Disallow: /
117
118 User-agent: EmailWolf
119 Disallow: /
120
121 User-agent: ExtractorPro
122 Disallow: /
123
124 User-agent: CopyRightCheck
125 Disallow: /
126
127 User-agent: Crescent
128 Disallow: /
129
130 User-agent: SiteSnagger
131 Disallow: /
132
133 User-agent: ProWebWalker
134 Disallow: /
135
136 User-agent: CheeseBot
137 Disallow: /
138
139 User-agent: LNSpiderguy
140 Disallow: /
141
142 User-agent: Teleport
143 Disallow: /
144
145 User-agent: TeleportPro
146 Disallow: /
147
148 User-agent: MIIxpc
149 Disallow: /
150
151 User-agent: Telesoft
152 Disallow: /
153
154 User-agent: Website Quester
155 Disallow: /
156
157 User-agent: moget/2.1
158 Disallow: /
159
160 User-agent: WebZip/4.0
161 Disallow: /
162
163 User-agent: WebStripper
164 Disallow: /
165
166 User-agent: WebSauger
167 Disallow: /
168
169 User-agent: WebCopier
170 Disallow: /
171
172 User-agent: NetAnts
173 Disallow: /
174
175 User-agent: Mister PiX
176 Disallow: /
177
178 User-agent: WebAuto
179 Disallow: /
180
181 User-agent: TheNomad
182 Disallow: /
183
184 User-agent: WWW-Collector-E
185 Disallow: /
186
187 User-agent: RMA
188 Disallow: /
189
190 User-agent: libWeb/clsHTTP
191 Disallow: /
192
193 User-agent: asterias
194 Disallow: /
195
196 User-agent: httplib
197 Disallow: /
198
199 User-agent: turingos
200 Disallow: /
201
202 User-agent: spanner
203 Disallow: /
204
205 User-agent: InfoNaviRobot
206 Disallow: /
207
208 User-agent: Harvest/1.5
209 Disallow: /
210
211 User-agent: Bullseye/1.0
212 Disallow: /
213
214 User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)
215 Disallow: /
216
217 User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
218 Disallow: /
219
220 User-agent: CherryPickerSE/1.0
221 Disallow: /
222
223 User-agent: CherryPickerElite/1.0
224 Disallow: /
225
226 User-agent: WebBandit/3.50
227 Disallow: /
228
229 User-agent: NICErsPRO
230 Disallow: /
231
232 User-agent: Microsoft URL Control - 5.01.4511
233 Disallow: /
234
235 User-agent: DittoSpyder
236 Disallow: /
237
238 User-agent: Foobot
239 Disallow: /
240
241 User-agent: WebmasterWorldForumBot
242 Disallow: /
243
244 User-agent: SpankBot
245 Disallow: /
246
247 User-agent: BotALot
248 Disallow: /
249
250 User-agent: lwp-trivial/1.34
251 Disallow: /
252
253 User-agent: lwp-trivial
254 Disallow: /
255
256 User-agent: BunnySlippers
257 Disallow: /
258
259 User-agent: Microsoft URL Control - 6.00.8169
260 Disallow: /
261
262 User-agent: URLy Warning
263 Disallow: /
264
265 User-agent: Wget/1.6
266 Disallow: /
267
268 User-agent: Wget/1.5.3
269 Disallow: /
270
271 User-agent: Wget
272 Disallow: /
273
274 User-agent: LinkWalker
275 Disallow: /
276
277 User-agent: cosmos
278 Disallow: /
279
280 User-agent: moget
281 Disallow: /
282
283 User-agent: hloader
284 Disallow: /
285
286 User-agent: humanlinks
287 Disallow: /
288
289 User-agent: LinkextractorPro
290 Disallow: /
291
292 User-agent: Offline Explorer
293 Disallow: /
294
295 User-agent: Mata Hari
296 Disallow: /
297
298 User-agent: LexiBot
299 Disallow: /
300
301 User-agent: Web Image Collector
302 Disallow: /
303
304 User-agent: The Intraformant
305 Disallow: /
306
307 User-agent: True_Robot/1.0
308 Disallow: /
309
310 User-agent: True_Robot
311 Disallow: /
312
313 User-agent: BlowFish/1.0
314 Disallow: /
315
316 User-agent: JennyBot
317 Disallow: /
318
319 User-agent: MIIxpc/4.2
320 Disallow: /
321
322 User-agent: BuiltBotTough
323 Disallow: /
324
325 User-agent: ProPowerBot/2.14
326 Disallow: /
327
328 User-agent: BackDoorBot/1.0
329 Disallow: /
330
331 User-agent: toCrawl/UrlDispatcher
332 Disallow: /
333
334 User-agent: WebEnhancer
335 Disallow: /
336
337 User-agent: suzuran
338 Disallow: /
339
340 User-agent: VCI WebViewer VCI WebViewer Win32
341 Disallow: /
342
343 User-agent: VCI
344 Disallow: /
345
346 User-agent: Szukacz/1.4 
347 Disallow: /
348
349 User-agent: QueryN Metasearch
350 Disallow: /
351
352 User-agent: Openfind data gathere
353 Disallow: /
354
355 User-agent: Openfind 
356 Disallow: /
357
358 User-agent: Xenu's Link Sleuth 1.1c
359 Disallow: /
360
361 User-agent: Xenu's
362 Disallow: /
363
364 User-agent: Zeus
365 Disallow: /
366
367 User-agent: RepoMonkey Bait & Tackle/v1.01
368 Disallow: /
369
370 User-agent: RepoMonkey
371 Disallow: /
372
373 User-agent: Microsoft URL Control
374 Disallow: /
375
376 User-agent: Openbot
377 Disallow: /
378
379 User-agent: URL Control
380 Disallow: /
381
382 User-agent: Zeus Link Scout
383 Disallow: /
384
385 User-agent: Zeus 32297 Webster Pro V2.9 Win32
386 Disallow: /
387
388 User-agent: Webster Pro
389 Disallow: /
390
391 User-agent: EroCrawler
392 Disallow: /
393
394 User-agent: LinkScan/8.1a Unix
395 Disallow: /
396
397 User-agent: Keyword Density/0.9
398 Disallow: /
399
400 User-agent: Kenjin Spider
401 Disallow: /
402
403 User-agent: Iron33/1.0.2
404 Disallow: /
405
406 User-agent: Bookmark search tool
407 Disallow: /
408
409 User-agent: GetRight/4.2
410 Disallow: /
411
412 User-agent: FairAd Client
413 Disallow: /
414
415 User-agent: Gaisbot
416 Disallow: /
417
418 User-agent: Aqua_Products
419 Disallow: /
420
421 User-agent: Radiation Retriever 1.1
422 Disallow: /
423
424 User-agent: Flaming AttackBot
425 Disallow: /
426
427 User-agent: Oracle Ultra Search
428 Disallow: /
429
430 User-agent: MSIECrawler
431 Disallow: /
432
433 User-agent: PerMan
434 Disallow: /
435
436 User-agent: searchpreview
437 Disallow: /
438
439 # This is for my web pages using spider-trap
440 User-agent: *
441 Disallow: /guestbook/
442 Disallow: /blacklist.txt
443 Disallow: /whitelist.txt
444
445 User-agent: Googlebot
446 Disallow: /guestbook/
447 Disallow: /blacklist.txt
448 Disallow: /whitelist.txt