introduced isSqlResult() which encapsulates checking MySQL/i results
[mailer.git] / robots.txt
1 # Generic exclusion (makes never sense)
2 User-agent: *
3 Disallow: /admin.php
4 Disallow: /agb.php
5 Disallow: /ajax.php
6 Disallow: /api.php
7 Disallow: /autoreg.php
8 Disallow: /beg.php
9 Disallow: /birthday_confirm.php
10 Disallow: /click.php
11 Disallow: /confirm.php
12 Disallow: /css.php
13 Disallow: /debug.php
14 Disallow: /doubler.php
15 Disallow: /img.php
16 Disallow: /install.php
17 Disallow: /js.php
18 Disallow: /lead-confirm.php
19 Disallow: /login.php
20 Disallow: /mailid.php
21 Disallow: /mailid_top.php
22 Disallow: /network-vcheck.php
23 Disallow: /network-delurl.php
24 Disallow: /ref.php
25 Disallow: /show_bonus.php
26 Disallow: /sponsor_confirm.php
27 Disallow: /sponsor_ref.php
28 Disallow: /surfbar.php
29 Disallow: /view.php
30 Disallow: /css/
31 Disallow: /inc/
32 Disallow: /images/
33 Disallow: /templates/
34 Disallow: /cms/index/wht/impressum
35
36 # Exclusion for Google (he will not listen to '*' :( )
37 User-agent: Googlebot
38 Disallow: /admin.php
39 Disallow: /agb.php
40 Disallow: /ajax.php
41 Disallow: /api.php
42 Disallow: /autoreg.php
43 Disallow: /beg.php
44 Disallow: /birthday_confirm.php
45 Disallow: /click.php
46 Disallow: /confirm.php
47 Disallow: /css.php
48 Disallow: /debug.php
49 Disallow: /doubler.php
50 Disallow: /img.php
51 Disallow: /install.php
52 Disallow: /js.php
53 Disallow: /lead-confirm.php
54 Disallow: /login.php
55 Disallow: /mailid.php
56 Disallow: /mailid_top.php
57 Disallow: /network-vcheck.php
58 Disallow: /network-delurl.php
59 Disallow: /ref.php
60 Disallow: /show_bonus.php
61 Disallow: /sponsor_confirm.php
62 Disallow: /sponsor_ref.php
63 Disallow: /surfbar.php
64 Disallow: /view.php
65 Disallow: /css/
66 Disallow: /inc/
67 Disallow: /images/
68 Disallow: /templates/
69 Disallow: /cms/index/wht/impressum
70
71 # A lot "bot" you may don't like as I do
72 User-agent: grub-client
73 Disallow: /
74
75 User-agent: grub
76 Disallow: /
77
78 User-agent: looksmart
79 Disallow: /
80
81 User-agent: WebZip
82 Disallow: /
83
84 User-agent: larbin
85 Disallow: /
86
87 User-agent: b2w/0.1
88 Disallow: /
89
90 User-agent: psbot
91 Disallow: /
92
93 User-agent: Python-urllib
94 Disallow: /
95
96 User-agent: NetMechanic
97 Disallow: /
98
99 User-agent: URL_Spider_Pro
100 Disallow: /
101
102 User-agent: CherryPicker
103 Disallow: /
104
105 User-agent: EmailCollector
106 Disallow: /
107
108 User-agent: EmailSiphon
109 Disallow: /
110
111 User-agent: WebBandit
112 Disallow: /
113
114 User-agent: EmailWolf
115 Disallow: /
116
117 User-agent: ExtractorPro
118 Disallow: /
119
120 User-agent: CopyRightCheck
121 Disallow: /
122
123 User-agent: Crescent
124 Disallow: /
125
126 User-agent: SiteSnagger
127 Disallow: /
128
129 User-agent: ProWebWalker
130 Disallow: /
131
132 User-agent: CheeseBot
133 Disallow: /
134
135 User-agent: LNSpiderguy
136 Disallow: /
137
138 User-agent: Teleport
139 Disallow: /
140
141 User-agent: TeleportPro
142 Disallow: /
143
144 User-agent: MIIxpc
145 Disallow: /
146
147 User-agent: Telesoft
148 Disallow: /
149
150 User-agent: Website Quester
151 Disallow: /
152
153 User-agent: moget/2.1
154 Disallow: /
155
156 User-agent: WebZip/4.0
157 Disallow: /
158
159 User-agent: WebStripper
160 Disallow: /
161
162 User-agent: WebSauger
163 Disallow: /
164
165 User-agent: WebCopier
166 Disallow: /
167
168 User-agent: NetAnts
169 Disallow: /
170
171 User-agent: Mister PiX
172 Disallow: /
173
174 User-agent: WebAuto
175 Disallow: /
176
177 User-agent: TheNomad
178 Disallow: /
179
180 User-agent: WWW-Collector-E
181 Disallow: /
182
183 User-agent: RMA
184 Disallow: /
185
186 User-agent: libWeb/clsHTTP
187 Disallow: /
188
189 User-agent: asterias
190 Disallow: /
191
192 User-agent: httplib
193 Disallow: /
194
195 User-agent: turingos
196 Disallow: /
197
198 User-agent: spanner
199 Disallow: /
200
201 User-agent: InfoNaviRobot
202 Disallow: /
203
204 User-agent: Harvest/1.5
205 Disallow: /
206
207 User-agent: Bullseye/1.0
208 Disallow: /
209
210 User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)
211 Disallow: /
212
213 User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
214 Disallow: /
215
216 User-agent: CherryPickerSE/1.0
217 Disallow: /
218
219 User-agent: CherryPickerElite/1.0
220 Disallow: /
221
222 User-agent: WebBandit/3.50
223 Disallow: /
224
225 User-agent: NICErsPRO
226 Disallow: /
227
228 User-agent: Microsoft URL Control - 5.01.4511
229 Disallow: /
230
231 User-agent: DittoSpyder
232 Disallow: /
233
234 User-agent: Foobot
235 Disallow: /
236
237 User-agent: WebmasterWorldForumBot
238 Disallow: /
239
240 User-agent: SpankBot
241 Disallow: /
242
243 User-agent: BotALot
244 Disallow: /
245
246 User-agent: lwp-trivial/1.34
247 Disallow: /
248
249 User-agent: lwp-trivial
250 Disallow: /
251
252 User-agent: BunnySlippers
253 Disallow: /
254
255 User-agent: Microsoft URL Control - 6.00.8169
256 Disallow: /
257
258 User-agent: URLy Warning
259 Disallow: /
260
261 User-agent: Wget/1.6
262 Disallow: /
263
264 User-agent: Wget/1.5.3
265 Disallow: /
266
267 User-agent: Wget
268 Disallow: /
269
270 User-agent: LinkWalker
271 Disallow: /
272
273 User-agent: cosmos
274 Disallow: /
275
276 User-agent: moget
277 Disallow: /
278
279 User-agent: hloader
280 Disallow: /
281
282 User-agent: humanlinks
283 Disallow: /
284
285 User-agent: LinkextractorPro
286 Disallow: /
287
288 User-agent: Offline Explorer
289 Disallow: /
290
291 User-agent: Mata Hari
292 Disallow: /
293
294 User-agent: LexiBot
295 Disallow: /
296
297 User-agent: Web Image Collector
298 Disallow: /
299
300 User-agent: The Intraformant
301 Disallow: /
302
303 User-agent: True_Robot/1.0
304 Disallow: /
305
306 User-agent: True_Robot
307 Disallow: /
308
309 User-agent: BlowFish/1.0
310 Disallow: /
311
312 User-agent: JennyBot
313 Disallow: /
314
315 User-agent: MIIxpc/4.2
316 Disallow: /
317
318 User-agent: BuiltBotTough
319 Disallow: /
320
321 User-agent: ProPowerBot/2.14
322 Disallow: /
323
324 User-agent: BackDoorBot/1.0
325 Disallow: /
326
327 User-agent: toCrawl/UrlDispatcher
328 Disallow: /
329
330 User-agent: WebEnhancer
331 Disallow: /
332
333 User-agent: suzuran
334 Disallow: /
335
336 User-agent: VCI WebViewer VCI WebViewer Win32
337 Disallow: /
338
339 User-agent: VCI
340 Disallow: /
341
342 User-agent: Szukacz/1.4 
343 Disallow: /
344
345 User-agent: QueryN Metasearch
346 Disallow: /
347
348 User-agent: Openfind data gathere
349 Disallow: /
350
351 User-agent: Openfind 
352 Disallow: /
353
354 User-agent: Xenu's Link Sleuth 1.1c
355 Disallow: /
356
357 User-agent: Xenu's
358 Disallow: /
359
360 User-agent: Zeus
361 Disallow: /
362
363 User-agent: RepoMonkey Bait & Tackle/v1.01
364 Disallow: /
365
366 User-agent: RepoMonkey
367 Disallow: /
368
369 User-agent: Microsoft URL Control
370 Disallow: /
371
372 User-agent: Openbot
373 Disallow: /
374
375 User-agent: URL Control
376 Disallow: /
377
378 User-agent: Zeus Link Scout
379 Disallow: /
380
381 User-agent: Zeus 32297 Webster Pro V2.9 Win32
382 Disallow: /
383
384 User-agent: Webster Pro
385 Disallow: /
386
387 User-agent: EroCrawler
388 Disallow: /
389
390 User-agent: LinkScan/8.1a Unix
391 Disallow: /
392
393 User-agent: Keyword Density/0.9
394 Disallow: /
395
396 User-agent: Kenjin Spider
397 Disallow: /
398
399 User-agent: Iron33/1.0.2
400 Disallow: /
401
402 User-agent: Bookmark search tool
403 Disallow: /
404
405 User-agent: GetRight/4.2
406 Disallow: /
407
408 User-agent: FairAd Client
409 Disallow: /
410
411 User-agent: Gaisbot
412 Disallow: /
413
414 User-agent: Aqua_Products
415 Disallow: /
416
417 User-agent: Radiation Retriever 1.1
418 Disallow: /
419
420 User-agent: Flaming AttackBot
421 Disallow: /
422
423 User-agent: Oracle Ultra Search
424 Disallow: /
425
426 User-agent: MSIECrawler
427 Disallow: /
428
429 User-agent: PerMan
430 Disallow: /
431
432 User-agent: searchpreview
433 Disallow: /
434
435 # This is for my web pages using spider-trap
436 User-agent: *
437 Disallow: /guestbook/
438 Disallow: /blacklist.txt
439 Disallow: /whitelist.txt
440
441 User-agent: Googlebot
442 Disallow: /guestbook/
443 Disallow: /blacklist.txt
444 Disallow: /whitelist.txt