robots.txt

   1 # Generic exclusion (makes never sense)
   2 User-agent: *
   3 Disallow: /admin.php
   4 Disallow: /agb.php
   5 Disallow: /ajax.php
   6 Disallow: /api.php
   7 Disallow: /autoreg.php
   8 Disallow: /beg.php
   9 Disallow: /birthday_confirm.php
  10 Disallow: /click.php
  11 Disallow: /confirm.php
  12 Disallow: /css.php
  13 Disallow: /debug.php
  14 Disallow: /doubler.php
  15 Disallow: /img.php
  16 Disallow: /install.php
  17 Disallow: /js.php
  18 Disallow: /lead-confirm.php
  19 Disallow: /login.php
  20 Disallow: /mailid.php
  21 Disallow: /mailid_top.php
  22 Disallow: /network-vcheck.php
  23 Disallow: /network-delurl.php
  24 Disallow: /ref.php
  25 Disallow: /show_bonus.php
  26 Disallow: /sponsor_confirm.php
  27 Disallow: /sponsor_ref.php
  28 Disallow: /surfbar.php
  29 Disallow: /view.php
  30 Disallow: /assets/
  31 Disallow: /css/
  32 Disallow: /js/
  33 Disallow: /inc/
  34 Disallow: /images/
  35 Disallow: /templates/
  36 Disallow: /cms/index/wht/impressum
  37
  38 # Exclusion for Google (he will not listen to '*' :( )
  39 User-agent: Googlebot
  40 Disallow: /admin.php
  41 Disallow: /agb.php
  42 Disallow: /ajax.php
  43 Disallow: /api.php
  44 Disallow: /autoreg.php
  45 Disallow: /beg.php
  46 Disallow: /birthday_confirm.php
  47 Disallow: /click.php
  48 Disallow: /confirm.php
  49 Disallow: /css.php
  50 Disallow: /debug.php
  51 Disallow: /doubler.php
  52 Disallow: /img.php
  53 Disallow: /install.php
  54 Disallow: /js.php
  55 Disallow: /lead-confirm.php
  56 Disallow: /login.php
  57 Disallow: /mailid.php
  58 Disallow: /mailid_top.php
  59 Disallow: /network-vcheck.php
  60 Disallow: /network-delurl.php
  61 Disallow: /ref.php
  62 Disallow: /show_bonus.php
  63 Disallow: /sponsor_confirm.php
  64 Disallow: /sponsor_ref.php
  65 Disallow: /surfbar.php
  66 Disallow: /view.php
  67 Disallow: /assets/
  68 Disallow: /css/
  69 Disallow: /js/
  70 Disallow: /inc/
  71 Disallow: /images/
  72 Disallow: /templates/
  73 Disallow: /cms/index/wht/impressum
  74
  75 # A lot "bot" you may don't like as I do
  76 User-agent: grub-client
  77 Disallow: /
  78
  79 User-agent: grub
  80 Disallow: /
  81
  82 User-agent: looksmart
  83 Disallow: /
  84
  85 User-agent: WebZip
  86 Disallow: /
  87
  88 User-agent: larbin
  89 Disallow: /
  90
  91 User-agent: b2w/0.1
  92 Disallow: /
  93
  94 User-agent: psbot
  95 Disallow: /
  96
  97 User-agent: Python-urllib
  98 Disallow: /
  99
 100 User-agent: NetMechanic
 101 Disallow: /
 102
 103 User-agent: URL_Spider_Pro
 104 Disallow: /
 105
 106 User-agent: CherryPicker
 107 Disallow: /
 108
 109 User-agent: EmailCollector
 110 Disallow: /
 111
 112 User-agent: EmailSiphon
 113 Disallow: /
 114
 115 User-agent: WebBandit
 116 Disallow: /
 117
 118 User-agent: EmailWolf
 119 Disallow: /
 120
 121 User-agent: ExtractorPro
 122 Disallow: /
 123
 124 User-agent: CopyRightCheck
 125 Disallow: /
 126
 127 User-agent: Crescent
 128 Disallow: /
 129
 130 User-agent: SiteSnagger
 131 Disallow: /
 132
 133 User-agent: ProWebWalker
 134 Disallow: /
 135
 136 User-agent: CheeseBot
 137 Disallow: /
 138
 139 User-agent: LNSpiderguy
 140 Disallow: /
 141
 142 User-agent: Teleport
 143 Disallow: /
 144
 145 User-agent: TeleportPro
 146 Disallow: /
 147
 148 User-agent: MIIxpc
 149 Disallow: /
 150
 151 User-agent: Telesoft
 152 Disallow: /
 153
 154 User-agent: Website Quester
 155 Disallow: /
 156
 157 User-agent: moget/2.1
 158 Disallow: /
 159
 160 User-agent: WebZip/4.0
 161 Disallow: /
 162
 163 User-agent: WebStripper
 164 Disallow: /
 165
 166 User-agent: WebSauger
 167 Disallow: /
 168
 169 User-agent: WebCopier
 170 Disallow: /
 171
 172 User-agent: NetAnts
 173 Disallow: /
 174
 175 User-agent: Mister PiX
 176 Disallow: /
 177
 178 User-agent: WebAuto
 179 Disallow: /
 180
 181 User-agent: TheNomad
 182 Disallow: /
 183
 184 User-agent: WWW-Collector-E
 185 Disallow: /
 186
 187 User-agent: RMA
 188 Disallow: /
 189
 190 User-agent: libWeb/clsHTTP
 191 Disallow: /
 192
 193 User-agent: asterias
 194 Disallow: /
 195
 196 User-agent: httplib
 197 Disallow: /
 198
 199 User-agent: turingos
 200 Disallow: /
 201
 202 User-agent: spanner
 203 Disallow: /
 204
 205 User-agent: InfoNaviRobot
 206 Disallow: /
 207
 208 User-agent: Harvest/1.5
 209 Disallow: /
 210
 211 User-agent: Bullseye/1.0
 212 Disallow: /
 213
 214 User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)
 215 Disallow: /
 216
 217 User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
 218 Disallow: /
 219
 220 User-agent: CherryPickerSE/1.0
 221 Disallow: /
 222
 223 User-agent: CherryPickerElite/1.0
 224 Disallow: /
 225
 226 User-agent: WebBandit/3.50
 227 Disallow: /
 228
 229 User-agent: NICErsPRO
 230 Disallow: /
 231
 232 User-agent: Microsoft URL Control - 5.01.4511
 233 Disallow: /
 234
 235 User-agent: DittoSpyder
 236 Disallow: /
 237
 238 User-agent: Foobot
 239 Disallow: /
 240
 241 User-agent: WebmasterWorldForumBot
 242 Disallow: /
 243
 244 User-agent: SpankBot
 245 Disallow: /
 246
 247 User-agent: BotALot
 248 Disallow: /
 249
 250 User-agent: lwp-trivial/1.34
 251 Disallow: /
 252
 253 User-agent: lwp-trivial
 254 Disallow: /
 255
 256 User-agent: BunnySlippers
 257 Disallow: /
 258
 259 User-agent: Microsoft URL Control - 6.00.8169
 260 Disallow: /
 261
 262 User-agent: URLy Warning
 263 Disallow: /
 264
 265 User-agent: Wget/1.6
 266 Disallow: /
 267
 268 User-agent: Wget/1.5.3
 269 Disallow: /
 270
 271 User-agent: Wget
 272 Disallow: /
 273
 274 User-agent: LinkWalker
 275 Disallow: /
 276
 277 User-agent: cosmos
 278 Disallow: /
 279
 280 User-agent: moget
 281 Disallow: /
 282
 283 User-agent: hloader
 284 Disallow: /
 285
 286 User-agent: humanlinks
 287 Disallow: /
 288
 289 User-agent: LinkextractorPro
 290 Disallow: /
 291
 292 User-agent: Offline Explorer
 293 Disallow: /
 294
 295 User-agent: Mata Hari
 296 Disallow: /
 297
 298 User-agent: LexiBot
 299 Disallow: /
 300
 301 User-agent: Web Image Collector
 302 Disallow: /
 303
 304 User-agent: The Intraformant
 305 Disallow: /
 306
 307 User-agent: True_Robot/1.0
 308 Disallow: /
 309
 310 User-agent: True_Robot
 311 Disallow: /
 312
 313 User-agent: BlowFish/1.0
 314 Disallow: /
 315
 316 User-agent: JennyBot
 317 Disallow: /
 318
 319 User-agent: MIIxpc/4.2
 320 Disallow: /
 321
 322 User-agent: BuiltBotTough
 323 Disallow: /
 324
 325 User-agent: ProPowerBot/2.14
 326 Disallow: /
 327
 328 User-agent: BackDoorBot/1.0
 329 Disallow: /
 330
 331 User-agent: toCrawl/UrlDispatcher
 332 Disallow: /
 333
 334 User-agent: WebEnhancer
 335 Disallow: /
 336
 337 User-agent: suzuran
 338 Disallow: /
 339
 340 User-agent: VCI WebViewer VCI WebViewer Win32
 341 Disallow: /
 342
 343 User-agent: VCI
 344 Disallow: /
 345
 346 User-agent: Szukacz/1.4
 347 Disallow: /
 348
 349 User-agent: QueryN Metasearch
 350 Disallow: /
 351
 352 User-agent: Openfind data gathere
 353 Disallow: /
 354
 355 User-agent: Openfind
 356 Disallow: /
 357
 358 User-agent: Xenu's Link Sleuth 1.1c
 359 Disallow: /
 360
 361 User-agent: Xenu's
 362 Disallow: /
 363
 364 User-agent: Zeus
 365 Disallow: /
 366
 367 User-agent: RepoMonkey Bait & Tackle/v1.01
 368 Disallow: /
 369
 370 User-agent: RepoMonkey
 371 Disallow: /
 372
 373 User-agent: Microsoft URL Control
 374 Disallow: /
 375
 376 User-agent: Openbot
 377 Disallow: /
 378
 379 User-agent: URL Control
 380 Disallow: /
 381
 382 User-agent: Zeus Link Scout
 383 Disallow: /
 384
 385 User-agent: Zeus 32297 Webster Pro V2.9 Win32
 386 Disallow: /
 387
 388 User-agent: Webster Pro
 389 Disallow: /
 390
 391 User-agent: EroCrawler
 392 Disallow: /
 393
 394 User-agent: LinkScan/8.1a Unix
 395 Disallow: /
 396
 397 User-agent: Keyword Density/0.9
 398 Disallow: /
 399
 400 User-agent: Kenjin Spider
 401 Disallow: /
 402
 403 User-agent: Iron33/1.0.2
 404 Disallow: /
 405
 406 User-agent: Bookmark search tool
 407 Disallow: /
 408
 409 User-agent: GetRight/4.2
 410 Disallow: /
 411
 412 User-agent: FairAd Client
 413 Disallow: /
 414
 415 User-agent: Gaisbot
 416 Disallow: /
 417
 418 User-agent: Aqua_Products
 419 Disallow: /
 420
 421 User-agent: Radiation Retriever 1.1
 422 Disallow: /
 423
 424 User-agent: Flaming AttackBot
 425 Disallow: /
 426
 427 User-agent: Oracle Ultra Search
 428 Disallow: /
 429
 430 User-agent: MSIECrawler
 431 Disallow: /
 432
 433 User-agent: PerMan
 434 Disallow: /
 435
 436 User-agent: searchpreview
 437 Disallow: /
 438
 439 # This is for my web pages using spider-trap
 440 User-agent: *
 441 Disallow: /guestbook/
 442 Disallow: /blacklist.txt
 443 Disallow: /whitelist.txt
 444
 445 User-agent: Googlebot
 446 Disallow: /guestbook/
 447 Disallow: /blacklist.txt
 448 Disallow: /whitelist.txt