robots.txt

   1 # Generic exclusion (makes never sense)
   2 User-agent: *
   3 Disallow: /admin.php
   4 Disallow: /agb.php
   5 Disallow: /ajax.php
   6 Disallow: /api.php
   7 Disallow: /autoreg.php
   8 Disallow: /beg.php
   9 Disallow: /birthday_confirm.php
  10 Disallow: /click.php
  11 Disallow: /confirm.php
  12 Disallow: /css.php
  13 Disallow: /debug.php
  14 Disallow: /doubler.php
  15 Disallow: /img.php
  16 Disallow: /install.php
  17 Disallow: /js.php
  18 Disallow: /lead-confirm.php
  19 Disallow: /login.php
  20 Disallow: /mailid.php
  21 Disallow: /mailid_top.php
  22 Disallow: /network-vcheck.php
  23 Disallow: /network-delurl.php
  24 Disallow: /ref.php
  25 Disallow: /register.php
  26 Disallow: /show_bonus.php
  27 Disallow: /sponsor_confirm.php
  28 Disallow: /sponsor_ref.php
  29 Disallow: /surfbar.php
  30 Disallow: /view.php
  31 Disallow: /assets/
  32 Disallow: /css/
  33 Disallow: /js/
  34 Disallow: /inc/
  35 Disallow: /images/
  36 Disallow: /templates/
  37 Disallow: /cms/index/wht/impressum
  38
  39 # Exclusion for Google (he will not listen to '*' :( )
  40 User-agent: Googlebot
  41 Disallow: /admin.php
  42 Disallow: /agb.php
  43 Disallow: /ajax.php
  44 Disallow: /api.php
  45 Disallow: /autoreg.php
  46 Disallow: /beg.php
  47 Disallow: /birthday_confirm.php
  48 Disallow: /click.php
  49 Disallow: /confirm.php
  50 Disallow: /css.php
  51 Disallow: /debug.php
  52 Disallow: /doubler.php
  53 Disallow: /img.php
  54 Disallow: /install.php
  55 Disallow: /js.php
  56 Disallow: /lead-confirm.php
  57 Disallow: /login.php
  58 Disallow: /mailid.php
  59 Disallow: /mailid_top.php
  60 Disallow: /network-vcheck.php
  61 Disallow: /network-delurl.php
  62 Disallow: /ref.php
  63 Disallow: /register.php
  64 Disallow: /show_bonus.php
  65 Disallow: /sponsor_confirm.php
  66 Disallow: /sponsor_ref.php
  67 Disallow: /surfbar.php
  68 Disallow: /view.php
  69 Disallow: /assets/
  70 Disallow: /css/
  71 Disallow: /js/
  72 Disallow: /inc/
  73 Disallow: /images/
  74 Disallow: /templates/
  75 Disallow: /cms/index/wht/impressum
  76
  77 # A lot "bot" you may don't like as I do
  78 User-agent: grub-client
  79 Disallow: /
  80
  81 User-agent: grub
  82 Disallow: /
  83
  84 User-agent: looksmart
  85 Disallow: /
  86
  87 User-agent: WebZip
  88 Disallow: /
  89
  90 User-agent: larbin
  91 Disallow: /
  92
  93 User-agent: b2w/0.1
  94 Disallow: /
  95
  96 User-agent: psbot
  97 Disallow: /
  98
  99 User-agent: Python-urllib
 100 Disallow: /
 101
 102 User-agent: NetMechanic
 103 Disallow: /
 104
 105 User-agent: URL_Spider_Pro
 106 Disallow: /
 107
 108 User-agent: CherryPicker
 109 Disallow: /
 110
 111 User-agent: EmailCollector
 112 Disallow: /
 113
 114 User-agent: EmailSiphon
 115 Disallow: /
 116
 117 User-agent: WebBandit
 118 Disallow: /
 119
 120 User-agent: EmailWolf
 121 Disallow: /
 122
 123 User-agent: ExtractorPro
 124 Disallow: /
 125
 126 User-agent: CopyRightCheck
 127 Disallow: /
 128
 129 User-agent: Crescent
 130 Disallow: /
 131
 132 User-agent: SiteSnagger
 133 Disallow: /
 134
 135 User-agent: ProWebWalker
 136 Disallow: /
 137
 138 User-agent: CheeseBot
 139 Disallow: /
 140
 141 User-agent: LNSpiderguy
 142 Disallow: /
 143
 144 User-agent: Teleport
 145 Disallow: /
 146
 147 User-agent: TeleportPro
 148 Disallow: /
 149
 150 User-agent: MIIxpc
 151 Disallow: /
 152
 153 User-agent: Telesoft
 154 Disallow: /
 155
 156 User-agent: Website Quester
 157 Disallow: /
 158
 159 User-agent: moget/2.1
 160 Disallow: /
 161
 162 User-agent: WebZip/4.0
 163 Disallow: /
 164
 165 User-agent: WebStripper
 166 Disallow: /
 167
 168 User-agent: WebSauger
 169 Disallow: /
 170
 171 User-agent: WebCopier
 172 Disallow: /
 173
 174 User-agent: NetAnts
 175 Disallow: /
 176
 177 User-agent: Mister PiX
 178 Disallow: /
 179
 180 User-agent: WebAuto
 181 Disallow: /
 182
 183 User-agent: TheNomad
 184 Disallow: /
 185
 186 User-agent: WWW-Collector-E
 187 Disallow: /
 188
 189 User-agent: RMA
 190 Disallow: /
 191
 192 User-agent: libWeb/clsHTTP
 193 Disallow: /
 194
 195 User-agent: asterias
 196 Disallow: /
 197
 198 User-agent: httplib
 199 Disallow: /
 200
 201 User-agent: turingos
 202 Disallow: /
 203
 204 User-agent: spanner
 205 Disallow: /
 206
 207 User-agent: InfoNaviRobot
 208 Disallow: /
 209
 210 User-agent: Harvest/1.5
 211 Disallow: /
 212
 213 User-agent: Bullseye/1.0
 214 Disallow: /
 215
 216 User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)
 217 Disallow: /
 218
 219 User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
 220 Disallow: /
 221
 222 User-agent: CherryPickerSE/1.0
 223 Disallow: /
 224
 225 User-agent: CherryPickerElite/1.0
 226 Disallow: /
 227
 228 User-agent: WebBandit/3.50
 229 Disallow: /
 230
 231 User-agent: NICErsPRO
 232 Disallow: /
 233
 234 User-agent: Microsoft URL Control - 5.01.4511
 235 Disallow: /
 236
 237 User-agent: DittoSpyder
 238 Disallow: /
 239
 240 User-agent: Foobot
 241 Disallow: /
 242
 243 User-agent: WebmasterWorldForumBot
 244 Disallow: /
 245
 246 User-agent: SpankBot
 247 Disallow: /
 248
 249 User-agent: BotALot
 250 Disallow: /
 251
 252 User-agent: lwp-trivial/1.34
 253 Disallow: /
 254
 255 User-agent: lwp-trivial
 256 Disallow: /
 257
 258 User-agent: BunnySlippers
 259 Disallow: /
 260
 261 User-agent: Microsoft URL Control - 6.00.8169
 262 Disallow: /
 263
 264 User-agent: URLy Warning
 265 Disallow: /
 266
 267 User-agent: Wget/1.6
 268 Disallow: /
 269
 270 User-agent: Wget/1.5.3
 271 Disallow: /
 272
 273 User-agent: Wget
 274 Disallow: /
 275
 276 User-agent: LinkWalker
 277 Disallow: /
 278
 279 User-agent: cosmos
 280 Disallow: /
 281
 282 User-agent: moget
 283 Disallow: /
 284
 285 User-agent: hloader
 286 Disallow: /
 287
 288 User-agent: humanlinks
 289 Disallow: /
 290
 291 User-agent: LinkextractorPro
 292 Disallow: /
 293
 294 User-agent: Offline Explorer
 295 Disallow: /
 296
 297 User-agent: Mata Hari
 298 Disallow: /
 299
 300 User-agent: LexiBot
 301 Disallow: /
 302
 303 User-agent: Web Image Collector
 304 Disallow: /
 305
 306 User-agent: The Intraformant
 307 Disallow: /
 308
 309 User-agent: True_Robot/1.0
 310 Disallow: /
 311
 312 User-agent: True_Robot
 313 Disallow: /
 314
 315 User-agent: BlowFish/1.0
 316 Disallow: /
 317
 318 User-agent: JennyBot
 319 Disallow: /
 320
 321 User-agent: MIIxpc/4.2
 322 Disallow: /
 323
 324 User-agent: BuiltBotTough
 325 Disallow: /
 326
 327 User-agent: ProPowerBot/2.14
 328 Disallow: /
 329
 330 User-agent: BackDoorBot/1.0
 331 Disallow: /
 332
 333 User-agent: toCrawl/UrlDispatcher
 334 Disallow: /
 335
 336 User-agent: WebEnhancer
 337 Disallow: /
 338
 339 User-agent: suzuran
 340 Disallow: /
 341
 342 User-agent: VCI WebViewer VCI WebViewer Win32
 343 Disallow: /
 344
 345 User-agent: VCI
 346 Disallow: /
 347
 348 User-agent: Szukacz/1.4
 349 Disallow: /
 350
 351 User-agent: QueryN Metasearch
 352 Disallow: /
 353
 354 User-agent: Openfind data gathere
 355 Disallow: /
 356
 357 User-agent: Openfind
 358 Disallow: /
 359
 360 User-agent: Xenu's Link Sleuth 1.1c
 361 Disallow: /
 362
 363 User-agent: Xenu's
 364 Disallow: /
 365
 366 User-agent: Zeus
 367 Disallow: /
 368
 369 User-agent: RepoMonkey Bait & Tackle/v1.01
 370 Disallow: /
 371
 372 User-agent: RepoMonkey
 373 Disallow: /
 374
 375 User-agent: Microsoft URL Control
 376 Disallow: /
 377
 378 User-agent: Openbot
 379 Disallow: /
 380
 381 User-agent: URL Control
 382 Disallow: /
 383
 384 User-agent: Zeus Link Scout
 385 Disallow: /
 386
 387 User-agent: Zeus 32297 Webster Pro V2.9 Win32
 388 Disallow: /
 389
 390 User-agent: Webster Pro
 391 Disallow: /
 392
 393 User-agent: EroCrawler
 394 Disallow: /
 395
 396 User-agent: LinkScan/8.1a Unix
 397 Disallow: /
 398
 399 User-agent: Keyword Density/0.9
 400 Disallow: /
 401
 402 User-agent: Kenjin Spider
 403 Disallow: /
 404
 405 User-agent: Iron33/1.0.2
 406 Disallow: /
 407
 408 User-agent: Bookmark search tool
 409 Disallow: /
 410
 411 User-agent: GetRight/4.2
 412 Disallow: /
 413
 414 User-agent: FairAd Client
 415 Disallow: /
 416
 417 User-agent: Gaisbot
 418 Disallow: /
 419
 420 User-agent: Aqua_Products
 421 Disallow: /
 422
 423 User-agent: Radiation Retriever 1.1
 424 Disallow: /
 425
 426 User-agent: Flaming AttackBot
 427 Disallow: /
 428
 429 User-agent: Oracle Ultra Search
 430 Disallow: /
 431
 432 User-agent: MSIECrawler
 433 Disallow: /
 434
 435 User-agent: PerMan
 436 Disallow: /
 437
 438 User-agent: searchpreview
 439 Disallow: /
 440
 441 # This is for my web pages using spider-trap
 442 User-agent: *
 443 Disallow: /guestbook/
 444 Disallow: /blacklist.txt
 445 Disallow: /whitelist.txt
 446
 447 User-agent: Googlebot
 448 Disallow: /guestbook/
 449 Disallow: /blacklist.txt
 450 Disallow: /whitelist.txt