]> git.mxchange.org Git - friendica-addons.git/commitdiff
[blockbot] Update Composer dependency ahead of release
authorHypolite Petovan <hypolite@mrpetovan.com>
Wed, 20 Mar 2024 02:51:17 +0000 (22:51 -0400)
committerHypolite Petovan <hypolite@mrpetovan.com>
Wed, 20 Mar 2024 03:10:09 +0000 (23:10 -0400)
- Updating jaybizzle/crawler-detect (v1.2.80 => v1.2.116)

20 files changed:
blockbot/composer.json
blockbot/composer.lock
blockbot/vendor/composer/ClassLoader.php
blockbot/vendor/composer/autoload_real.php
blockbot/vendor/composer/installed.json
blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/php-cs-fixer.yml [new file with mode: 0644]
blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/test.yml [new file with mode: 0644]
blockbot/vendor/jaybizzle/crawler-detect/.php_cs.dist [new file with mode: 0644]
blockbot/vendor/jaybizzle/crawler-detect/LICENSE
blockbot/vendor/jaybizzle/crawler-detect/README.md
blockbot/vendor/jaybizzle/crawler-detect/composer.json
blockbot/vendor/jaybizzle/crawler-detect/export.php
blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.json
blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.txt
blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.json
blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.txt
blockbot/vendor/jaybizzle/crawler-detect/src/CrawlerDetect.php
blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/AbstractProvider.php
blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Crawlers.php
blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Exclusions.php

index f13a2d17312240642ea2142e76e18427af454abc..40583e209ddb5388f980de44f88ab2c989ff0662 100644 (file)
@@ -1,24 +1,24 @@
 {
-       "name": "friendica-addons/blockbot",
-       "description": "Blocking bots based on detecting bots/crawlers/spiders via the user agent and http_from header.",
-       "type": "friendica-addon",
-       "authors": [
-               {
-                       "name": "Philipp Holzer",
-                       "email": "admin@philipp.info",
-                       "homepage": "https://friendica.philipp.info/profile/nupplaphil",
-                       "role": "Developer"
-               }
-       ],
-       "require": {
-               "php": ">=5.6.0",
-               "jaybizzle/crawler-detect": "1.*"
-       },
-       "license": "3-clause BSD license",
-       "minimum-stability": "stable",
-       "config": {
-               "optimize-autoloader": true,
-               "autoloader-suffix": "BlockBotAddon",
-               "preferred-install": "dist"
-       }
+    "name": "friendica-addons/blockbot",
+    "description": "Blocking bots based on detecting bots/crawlers/spiders via the user agent and http_from header.",
+    "type": "friendica-addon",
+    "authors": [
+        {
+            "name": "Philipp Holzer",
+            "email": "admin@philipp.info",
+            "homepage": "https://friendica.philipp.info/profile/nupplaphil",
+            "role": "Developer"
+        }
+    ],
+    "require": {
+        "php": ">=5.6.0",
+        "jaybizzle/crawler-detect": "1.*"
+    },
+    "license": "3-clause BSD license",
+    "minimum-stability": "stable",
+    "config": {
+        "optimize-autoloader": true,
+        "autoloader-suffix": "BlockBotAddon",
+        "preferred-install": "dist"
+    }
 }
index 26b021b11f2ab3951624be3992d4103975e1b0f7..50f71a3790a29f9b91765ffe0e7e61773f641b77 100644 (file)
@@ -8,24 +8,23 @@
     "packages": [
         {
             "name": "jaybizzle/crawler-detect",
-            "version": "v1.2.80",
+            "version": "v1.2.116",
             "source": {
                 "type": "git",
                 "url": "https://github.com/JayBizzle/Crawler-Detect.git",
-                "reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847"
+                "reference": "97e9fe30219e60092e107651abb379a38b342921"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847",
-                "reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847",
+                "url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/97e9fe30219e60092e107651abb379a38b342921",
+                "reference": "97e9fe30219e60092e107651abb379a38b342921",
                 "shasum": ""
             },
             "require": {
                 "php": ">=5.3.0"
             },
             "require-dev": {
-                "phpunit/phpunit": "^4.8|^5.5|^6.5",
-                "satooshi/php-coveralls": "1.*"
+                "phpunit/phpunit": "^4.8|^5.5|^6.5|^9.4"
             },
             "type": "library",
             "autoload": {
@@ -53,7 +52,7 @@
                 "crawlerdetect",
                 "php crawler detect"
             ],
-            "time": "2019-04-05T19:52:02+00:00"
+            "time": "2023-07-21T15:49:49+00:00"
         }
     ],
     "packages-dev": [],
@@ -65,5 +64,6 @@
     "platform": {
         "php": ">=5.6.0"
     },
-    "platform-dev": []
+    "platform-dev": [],
+    "plugin-api-version": "1.1.0"
 }
index 95f7e0978bad11efb1f75f5049b0c2ac7ae1939b..03b9bb9c40cb86c2c2bbec2ce6ff0ddce9ad586c 100644 (file)
@@ -60,7 +60,7 @@ class ClassLoader
     public function getPrefixes()
     {
         if (!empty($this->prefixesPsr0)) {
-            return call_user_func_array('array_merge', $this->prefixesPsr0);
+            return call_user_func_array('array_merge', array_values($this->prefixesPsr0));
         }
 
         return array();
@@ -279,7 +279,7 @@ class ClassLoader
      */
     public function setApcuPrefix($apcuPrefix)
     {
-        $this->apcuPrefix = function_exists('apcu_fetch') && ini_get('apc.enabled') ? $apcuPrefix : null;
+        $this->apcuPrefix = function_exists('apcu_fetch') && filter_var(ini_get('apc.enabled'), FILTER_VALIDATE_BOOLEAN) ? $apcuPrefix : null;
     }
 
     /**
index ccb886a23a07914c04ff8f182c1429dda84f1d58..404079e0e7f318c5bd048b50e426c42594411791 100644 (file)
@@ -13,6 +13,9 @@ class ComposerAutoloaderInitBlockBotAddon
         }
     }
 
+    /**
+     * @return \Composer\Autoload\ClassLoader
+     */
     public static function getLoader()
     {
         if (null !== self::$loader) {
index d255d5738a3148e0351c368921646c5a17fb8231..db9064b6dd390925ea2d355c6aef6f8b439b2219 100644 (file)
@@ -1,27 +1,26 @@
 [
     {
         "name": "jaybizzle/crawler-detect",
-        "version": "v1.2.80",
-        "version_normalized": "1.2.80.0",
+        "version": "v1.2.116",
+        "version_normalized": "1.2.116.0",
         "source": {
             "type": "git",
             "url": "https://github.com/JayBizzle/Crawler-Detect.git",
-            "reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847"
+            "reference": "97e9fe30219e60092e107651abb379a38b342921"
         },
         "dist": {
             "type": "zip",
-            "url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847",
-            "reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847",
+            "url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/97e9fe30219e60092e107651abb379a38b342921",
+            "reference": "97e9fe30219e60092e107651abb379a38b342921",
             "shasum": ""
         },
         "require": {
             "php": ">=5.3.0"
         },
         "require-dev": {
-            "phpunit/phpunit": "^4.8|^5.5|^6.5",
-            "satooshi/php-coveralls": "1.*"
+            "phpunit/phpunit": "^4.8|^5.5|^6.5|^9.4"
         },
-        "time": "2019-04-05T19:52:02+00:00",
+        "time": "2023-07-21T15:49:49+00:00",
         "type": "library",
         "installation-source": "dist",
         "autoload": {
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/php-cs-fixer.yml b/blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/php-cs-fixer.yml
new file mode 100644 (file)
index 0000000..1c083c4
--- /dev/null
@@ -0,0 +1,23 @@
+name: Check & fix styling
+
+on: [ push ]
+
+jobs:
+    php-cs-fixer:
+        runs-on: ubuntu-latest
+
+        steps:
+            -   name: Checkout code
+                uses: actions/checkout@v2
+                with:
+                    ref: ${{ github.head_ref }}
+
+            -   name: Run PHP CS Fixer
+                uses: docker://oskarstark/php-cs-fixer-ga:2.18.6
+                with:
+                    args: --config=.php_cs.dist --allow-risky=yes
+
+            -   name: Commit changes
+                uses: stefanzweifel/git-auto-commit-action@v4
+                with:
+                    commit_message: Fix styling
\ No newline at end of file
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/test.yml b/blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/test.yml
new file mode 100644 (file)
index 0000000..2291111
--- /dev/null
@@ -0,0 +1,56 @@
+name: Test
+
+on:
+  push:
+    branches:
+      - "master"
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        php: [5.3, 5.4, 5.5, 5.6, 7.0, 7.1, 7.2, 7.3, 7.4, 8.0, 8.1, 8.2]
+
+    name: PHP:${{ matrix.php }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup PHP, with composer
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: ${{ matrix.php }}
+          tools: composer:v2
+          coverage: xdebug
+
+      - name: Get composer cache directory
+        id: composer-cache
+        run: |
+          echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT
+        shell: bash
+
+      - name: Cache composer dependencies
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.composer-cache.outputs.dir }}
+          key: dependencies-php-${{ matrix.php }}-composer-${{ hashFiles('composer.json') }}
+          restore-keys: dependencies-php-${{ matrix.php }}-composer-
+
+      - name: Install Composer dependencies
+        run: |
+          composer install --prefer-dist --no-interaction --no-suggest
+
+      - name: Run Unit tests
+        run: |
+          vendor/bin/phpunit --coverage-clover=tests/logs/clover.xml
+
+      - name: Upload coverage results to Coveralls
+        env:
+          COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          composer global require php-coveralls/php-coveralls "^1.0"
+          coveralls --coverage_clover=tests/logs/clover.xml -v
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/.php_cs.dist b/blockbot/vendor/jaybizzle/crawler-detect/.php_cs.dist
new file mode 100644 (file)
index 0000000..91c91af
--- /dev/null
@@ -0,0 +1,33 @@
+<?php
+
+$finder = Symfony\Component\Finder\Finder::create()
+    ->in([
+        __DIR__.'/src',
+        __DIR__.'/tests',
+    ])
+    ->name('*.php')
+    ->ignoreDotFiles(true)
+    ->ignoreVCS(true);
+
+return PhpCsFixer\Config::create()
+    ->setRules([
+        '@PSR2' => true,
+        'array_syntax' => ['syntax' => 'long'],
+        'ordered_imports' => ['sortAlgorithm' => 'alpha'],
+        'no_unused_imports' => true,
+        'not_operator_with_successor_space' => true,
+        'trailing_comma_in_multiline_array' => true,
+        'phpdoc_scalar' => true,
+        'unary_operator_spaces' => true,
+        'binary_operator_spaces' => true,
+        'blank_line_before_statement' => [
+            'statements' => ['break', 'continue', 'declare', 'return', 'throw', 'try'],
+        ],
+        'phpdoc_single_line_var_spacing' => true,
+        'phpdoc_var_without_name' => true,
+        'method_argument_space' => [
+            'on_multiline' => 'ensure_fully_multiline',
+            'keep_multiple_spaces_after_comma' => true,
+        ],
+    ])
+    ->setFinder($finder);
\ No newline at end of file
index 2f4e15e252caa1266a077ce2ffb4a844b811a17f..569c7b4fd7ad8aac975c277c92d7770fdb03a6c6 100644 (file)
@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015-2018 Mark Beech
+Copyright (c) 2015-2020 Mark Beech
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
index e7c25f3b32a34cae967266ac29f4968bcfcb9897..57ec8e8b5329c291b0e4b70e033b75d98ef8efc1 100644 (file)
@@ -1,24 +1,23 @@
-<p align="center"><a href="http://crawlerdetect.io/" target="_blank"><img src="https://cloud.githubusercontent.com/assets/340752/23082173/1bd1a396-f550-11e6-8aba-4d3c75edea2f.png" width="321" height="219" /></a><br><br>
-<a href="http://crawlerdetect.io/" target="_blank">crawlerdetect.io</a>
+<p align="center"><a href="https://crawlerdetect.io/" target="_blank"><img src="https://cloud.githubusercontent.com/assets/340752/23082173/1bd1a396-f550-11e6-8aba-4d3c75edea2f.png" width="321" height="219" /></a><br><br>
+<a href="https://crawlerdetect.io/" target="_blank">crawlerdetect.io</a>
 <br><br>
 </p>
-
 <p align="center">
-<a href="https://travis-ci.org/JayBizzle/Crawler-Detect"><img src="https://img.shields.io/travis/JayBizzle/Crawler-Detect/master.svg?style=flat-square" /></a>
+<a href="https://github.com/JayBizzle/Crawler-Detect/actions"><img alt="GitHub Workflow Status" src="https://img.shields.io/github/actions/workflow/status/JayBizzle/Crawler-Detect/test.yml?branch=master&style=flat-square"></a>
 <a href="https://packagist.org/packages/jaybizzle/crawler-detect"><img src="https://img.shields.io/packagist/dm/JayBizzle/Crawler-Detect.svg?style=flat-square" /></a>
-<a href="https://scrutinizer-ci.com/g/JayBizzle/Crawler-Detect/?branch=master"><img src="https://img.shields.io/scrutinizer/g/JayBizzle/Crawler-Detect.svg?style=flat-square" /></a>
 <a href="https://github.com/JayBizzle/Crawler-Detect"><img src="https://img.shields.io/badge/license-MIT-ff69b4.svg?style=flat-square" /></a>
 <a href="https://packagist.org/packages/jaybizzle/crawler-detect"><img src="https://img.shields.io/packagist/v/jaybizzle/Crawler-Detect.svg?style=flat-square" /></a>
-<a href="https://styleci.io/repos/32755917"><img src="https://styleci.io/repos/32755917/shield" /></a>
 <a href="https://coveralls.io/github/JayBizzle/Crawler-Detect"><img src="https://img.shields.io/coveralls/JayBizzle/Crawler-Detect/master.svg?style=flat-square" /></a>
 </p>
 
 ## About CrawlerDetect
 
-CrawlerDetect is a PHP class for detecting bots/crawlers/spiders via the user agent and http_from header. Currently able to detect 1,000's of bots/spiders/crawlers.
+CrawlerDetect is a PHP class for detecting bots/crawlers/spiders via the `user agent` and `http_from` header. Currently able to detect 1,000's of bots/spiders/crawlers.
 
 ### Installation
-Run `composer require jaybizzle/crawler-detect 1.*` or add `"jaybizzle/crawler-detect" :"1.*"` to your `composer.json`.
+```
+composer require jaybizzle/crawler-detect
+```
 
 ### Usage
 ```PHP
@@ -46,7 +45,7 @@ If you find a bot/spider/crawler user agent that CrawlerDetect fails to detect,
 Failing that, just create an issue with the user agent you have found, and we'll take it from there :)
 
 ### Laravel Package
-If you would like to use this with Laravel 4/5, please see [Laravel-Crawler-Detect](https://github.com/JayBizzle/Laravel-Crawler-Detect)
+If you would like to use this with Laravel, please see [Laravel-Crawler-Detect](https://github.com/JayBizzle/Laravel-Crawler-Detect)
 
 ### Symfony Bundle
 To use this library with Symfony 2/3/4, check out the [CrawlerDetectBundle](https://github.com/nicolasmure/CrawlerDetectBundle).
@@ -57,16 +56,21 @@ To use this library with the YII2 framework, check out [yii2-crawler-detect](htt
 ### ES6 Library
 To use this library with NodeJS or any ES6 application based, check out [es6-crawler-detect](https://github.com/JefferyHus/es6-crawler-detect).
 
+### Python Library
+To use this library in a Python project, check out [crawlerdetect](https://github.com/moskrc/CrawlerDetect).
+
+### JVM Library (written in Java)
+To use this library in a JVM project (including Java, Scala, Kotlin, etc.), check out [CrawlerDetect](https://github.com/nekosoftllc/crawler-detect).
+
 ### .NET Library
 To use this library in a .net standard (including .net core) based project, check out [NetCrawlerDetect](https://github.com/gplumb/NetCrawlerDetect).
 
-### Nette Extension
-To use this library with the Nette framework, checkout [NetteCrawlerDetect](https://github.com/JanGalek/Crawler-Detect).
-
 ### Ruby Gem
-
 To use this library with Ruby on Rails or any Ruby-based application, check out [crawler_detect](https://github.com/loadkpi/crawler_detect) gem.
 
+### Go Module
+To use this library with Go, check out the [crawlerdetect](https://github.com/x-way/crawlerdetect) module.
+
 _Parts of this class are based on the brilliant [MobileDetect](https://github.com/serbanghita/Mobile-Detect)_
 
 [![Analytics](https://ga-beacon.appspot.com/UA-72430465-1/Crawler-Detect/readme?pixel)](https://github.com/JayBizzle/Crawler-Detect)
index 0c0babe6c51823547b409374a9e28454757ca62c..4774117ec63730bf01ebb4c093bdac243ee17883 100755 (executable)
@@ -16,8 +16,7 @@
         "php": ">=5.3.0"
     },
     "require-dev": {
-        "phpunit/phpunit": "^4.8|^5.5|^6.5",
-        "satooshi/php-coveralls": "1.*"
+        "phpunit/phpunit": "^4.8|^5.5|^6.5|^9.4"
     },
     "autoload": {
         "psr-4": {
index 4c4b9d5d37c3f6bee790c0019faeaac9f7fb20ce..6c7459c480a227fb2c122fc8288a882be68f6586 100644 (file)
@@ -37,5 +37,5 @@ function outputJson($object)
 function outputTxt($object)
 {
     $className = (new ReflectionClass($object))->getShortName();
-    file_put_contents("raw/$className.txt", implode($object->getAll(), PHP_EOL));
+    file_put_contents("raw/$className.txt", implode(PHP_EOL, $object->getAll()));
 }
index a1e690ebfb416b3f69c52342421c1bd5afa9838a..003b87c8ca0957a843352080d7b04c095c4de8c3 100644 (file)
@@ -1 +1 @@
-[".*Java.*outbrain"," YLT","^b0t$","^bluefish ","^Calypso v\\\/","^COMODO DCV","^DangDang","^DavClnt","^FDM ","^git\\\/","^Goose\\\/","^Grabber","^HTTPClient\\\/","^Java\\\/","^Jeode\\\/","^Jetty\\\/","^Mail\\\/","^Mget","^Microsoft URL Control","^NG\\\/[0-9\\.]","^NING\\\/","^PHP\\\/[0-9]","^RMA\\\/","^Ruby|Ruby\\\/[0-9]","^VSE\\\/[0-9]","^WordPress\\.com","^XRL\\\/[0-9]","^ZmEu","008\\\/","13TABS","192\\.comAgent","2ip\\.ru","404enemy","7Siters","80legs","a\\.pr-cy\\.ru","a3logics\\.in","A6-Indexer","Abonti","Aboundex","aboutthedomain","Accoona-AI-Agent","acoon","acrylicapps\\.com\\\/pulp","Acunetix","AdAuth\\\/","adbeat","AddThis","ADmantX","AdminLabs","adressendeutschland","adscanner","Adstxtaggregator","agentslug","AHC","aihit","aiohttp\\\/","Airmail","akka-http\\\/","akula\\\/","alertra","alexa site audit","Alibaba\\.Security\\.Heimdall","Alligator","allloadin","AllSubmitter","alyze\\.info","amagit","Anarchie","AndroidDownloadManager","Anemone","AngleSharp","annotate_google","Ant\\.com","Anturis Agent","AnyEvent-HTTP\\\/","Apache Droid","Apache OpenOffice","Apache-HttpAsyncClient","Apache-HttpClient","ApacheBench","Apexoo","APIs-Google","AportWorm\\\/","AppBeat\\\/","AppEngine-Google","AppStoreScraperZ","Aprc\\\/[0-9]","Arachmo","arachnode","Arachnophilia","aria2","Arukereso","asafaweb","AskQuickly","Ask Jeeves","ASPSeek","Asterias","Astute","asynchttp","Attach","autocite","Autonomy","axios\\\/","B-l-i-t-z-B-O-T","Backlink-Ceck","backlink-check","BacklinkHttpStatus","BackStreet","BackWeb","Bad-Neighborhood","Badass","baidu\\.com","Bandit","basicstate","BatchFTP","Battleztar Bazinga","baypup\\\/","BazQux","BBBike","BCKLINKS","BDFetch","BegunAdvertising","Bidtellect","BigBozz","Bigfoot","biglotron","BingLocalSearch","BingPreview","binlar","biNu image cacher","Bitacle","biz_Directory","Black Hole","Blackboard Safeassign","BlackWidow","BlockNote\\.Net","Bloglines","Bloglovin","BlogPulseLive","BlogSearch","Blogtrottr","BlowFish","boitho\\.com-dc","BPImageWalker","Braintree-Webhooks","Branch Metrics API","Branch-Passthrough","Brandprotect","BrandVerity","Brandwatch","Brodie\\\/","Browsershots","BUbiNG","Buck\\\/","Buddy","BuiltWith","Bullseye","BunnySlippers","Burf Search","Butterfly\\\/","BuzzSumo","CAAM\\\/[0-9]","CakePHP","Calculon","Canary%20Mail","CaretNail","catexplorador","CC Metadata Scaper","Cegbfeieh","censys","Cerberian Drtrs","CERT\\.at-Statistics-Survey","cg-eye","changedetection","ChangesMeter","Charlotte","CheckHost","checkprivacy","CherryPicker","ChinaClaw","Chirp\\\/","chkme\\.com","Chlooe","Chromaxa","CirrusExplorer","CISPA Vulnerability Notification","Citoid","CJNetworkQuality","Clarsentia","clips\\.ua\\.ac\\.be","Cloud mapping","CloudEndure","CloudFlare-AlwaysOnline","Cloudinary","cmcm\\.com","coccoc","cognitiveseo","colly -","CommaFeed","Commons-HttpClient","commonscan","contactbigdatafr","contentkingapp","convera","CookieReports","copyright sheriff","CopyRightCheck","Copyscape","Cosmos4j\\.feedback","Covario-IDS","Crescent","Crowsnest","Criteo","CSHttp","curb","Curious George","curl","cuwhois\\\/","cybo\\.com","DAP\\\/NetHTTP","DareBoost","DatabaseDriverMysqli","DataCha0s","Datafeedwatch","Datanyze","DataparkSearch","dataprovider","DataXu","Daum(oa)?[ \\\/][0-9]","Demon","DeuSu","developers\\.google\\.com\\\/\\+\\\/web\\\/snippet\\\/","Devil","Digg","Digincore","DigitalPebble","Dirbuster","Discourse Forum Onebox","Disqus\\\/","Dispatch\\\/","DittoSpyder","dlvr","DMBrowser","DNSPod-reporting","docoloc","Dolphin http client","DomainAppender","Donuts Content Explorer","dotMailer content retrieval","dotSemantic","downforeveryoneorjustme","Download Wonder","downnotifier","DowntimeDetector","Drip","drupact","Drupal \\(\\+http:\\\/\\\/drupal\\.org\\\/\\)","DTS Agent","dubaiindex","EARTHCOM","Easy-Thumb","EasyDL","Ebingbong","ec2linkfinder","eCairn-Grabber","eCatch","ECCP","eContext\\\/","Ecxi","EirGrabber","ElectricMonk","elefent","EMail Exractor","EMail Wolf","EmailWolf","Embarcadero","Embed PHP Library","Embedly","endo\\\/","europarchive\\.org","evc-batch","EventMachine HttpClient","Everwall Link Expander","Evidon","Evrinid","ExactSearch","ExaleadCloudview","Excel\\\/","exif","Exploratodo","Express WebPictures","Extreme Picture Finder","EyeNetIE","ezooms","facebookexternalhit","facebookplatform","fairshare","Faraday v","fasthttp","Faveeo","Favicon downloader","faviconkit","faviconarchive","FavOrg","Feed Wrangler","Feedable\\\/","Feedbin","FeedBooster","FeedBucket","FeedBunch\\\/","FeedBurner","feeder","Feedly","FeedshowOnline","Feedspot","Feedwind\\\/","FeedZcollector","feeltiptop","Fetch API","Fetch\\\/[0-9]","Fever\\\/[0-9]","FHscan","Fimap","findlink","findthatfile","FlashGet","FlipboardBrowserProxy","FlipboardProxy","FlipboardRSS","Flock\\\/","fluffy","Flunky","flynxapp","forensiq","FoundSeoTool","http:\\\/\\\/www.neomo.de\\\/","free thumbnails","Freeuploader","Funnelback","G-i-g-a-b-o-t","g00g1e\\.net","ganarvisitas","geek-tools","Genieo","GentleSource","GetCode","Getintent","GetLinkInfo","getprismatic","GetRight","getroot","GetURLInfo\\\/","GetWeb","Ghost Inspector","GigablastOpenSource","GIS-LABS","github-camo","github\\.com","Go [\\d\\.]* package http","Go http package","Go-Ahead-Got-It","Go-http-client","Go!Zilla","gobyus","gofetch","GomezAgent","gooblog","Goodzer\\\/","Google AppsViewer","Google Desktop","Google favicon","Google Keyword Suggestion","Google Keyword Tool","Google Page Speed Insights","Google PP Default","Google Search Console","Google Web Preview","Google-Adwords","Google-Apps-Script","Google-Calendar-Importer","Google-HotelAdsVerifier","Google-HTTP-Java-Client","Google-Publisher-Plugin","Google-SearchByImage","Google-Site-Verification","Google-Structured-Data-Testing-Tool","Google-Youtube-Links","google-xrawler","GoogleDocs","GoogleHC\\\/","GoogleProducer","GoogleSites","Google-Transparency-Report","Gookey","GoScraper","GoSpotCheck","gosquared-thumbnailer","Gotit","GoZilla","grabify","GrabNet","Grafula","Grammarly","GrapeFX","GreatNews","Gregarius","GRequests","grokkit","grouphigh","grub-client","gSOAP\\\/","GT::WWW","GTmetrix","GuzzleHttp","gvfs\\\/","HAA(A)?RTLAND http client","Haansoft","hackney\\\/","Hadi Agent","HappyApps-WebCheck","Hatena","Havij","HeadlessChrome","HEADMasterSEO","HeartRails_Capture","help@dataminr\\.com","heritrix","historious","hkedcity","hledejLevne\\.cz","Hloader","HMView","Holmes","HonesoSearchEngine","HootSuite Image proxy","Hootsuite-WebFeed","hosterstats","HostTracker","ht:\\\/\\\/check","htdig","HTMLparser","htmlyse","HTTP Banner Detection","HTTP_Compression_Test","http_request2","http_requester","http-get","HTTP-Header-Abfrage","http-kit","http-request\\\/","HTTP-Tiny","HTTP::Lite","http\\.rb\\\/","http_get","HttpComponents","httphr","HTTPMon","httpRequest","httpscheck","httpssites_power","httpunit","HttpUrlConnection","httrack","huaweisymantec","HubSpot ","Humanlinks","i2kconnect\\\/","Iblog","ichiro","Id-search","IdeelaborPlagiaat","IDG Twitter Links Resolver","IDwhois\\\/","Iframely","igdeSpyder","IlTrovatore","Image Fetch","Image Sucker","ImageEngine\\\/","ImageVisu\\\/","Imagga","imagineeasy","imgsizer","InAGist","inbound\\.li parser","InDesign%20CC","Indy Library","InetURL","infegy","infohelfer","InfoTekies","InfoWizards Reciprocal Link","inpwrd\\.com","instabid","Instapaper","Integrity","integromedb","Intelliseek","InterGET","internet_archive","Internet Ninja","InternetSeer","internetVista monitor","intraVnews","IODC","IOI","iplabel","ips-agent","IPS\\\/[0-9]","IPWorks HTTP\\\/S Component","iqdb\\\/","Iria","Irokez","isitup\\.org","iskanie","isUp\\.li","iThemes Sync\\\/","iZSearch","JAHHO","janforman","Jaunt\\\/","Jbrofuzz","Jersey\\\/","JetCar","Jigsaw","Jobboerse","JobFeed discovery","Jobg8 URL Monitor","jobo","Jobrapido","Jobsearch1\\.5","JoinVision Generic","JolokiaPwn","Joomla","Jorgee","JS-Kit","JustView","Kaspersky Lab CFR link resolver","Kelny\\\/","Kerrigan\\\/","KeyCDN","Keyword Density","Keywords Research","KickFire","KimonoLabs\\\/","Kml-Google","knows\\.is","KOCMOHABT","kouio","kube-probe","kulturarw3","KumKie","L\\.webis","Larbin","Lavf\\\/","LeechFTP","LeechGet","letsencrypt","Lftp","LibVLC","LibWeb","Libwhisker","libwww","Licorne","Liferea\\\/","Lightspeedsystems","Lighthouse","Likse","Link Valet","link_thumbnailer","LinkAlarm\\\/","linkCheck","linkdex","LinkExaminer","linkfluence","linkpeek","LinkPreviewGenerator","LinkScan","LinksManager","LinkTiger","LinkWalker","Lipperhey","Litemage_walker","livedoor ScreenShot","LoadImpactRload","localsearch-web","LongURL API","looksystems\\.net","ltx71","lua-resty-http","lwp-request","lwp-trivial","LWP::Simple","lycos","LYT\\.SR","mabontland","Mag-Net","MagpieRSS","Mail\\.Ru","MailChimp","Majestic12","makecontact\\\/","Mandrill","MapperCmd","marketinggrader","MarkMonitor","MarkWatch","Mass Downloader","masscan\\\/","Mata Hari","Mediapartners-Google","mediawords","MegaIndex\\.ru","MeltwaterNews","Melvil Rawi","MemGator","Metaspinner","MetaURI","MFC_Tear_Sample","Microsearch","Microsoft Office ","Microsoft Outlook","Microsoft Windows Network Diagnostics","Microsoft-WebDAV-MiniRedir","Microsoft Data Access","MIDown tool","MIIxpc","Mindjet","Miniature\\.io","Miniflux","Mister PiX","mixdata dot com","mixed-content-scan","Mixmax-LinkPreview","mixnode","Mnogosearch","mogimogi","Mojeek","Mojolicious \\(Perl\\)","Monit\\\/","monitis","Monitority\\\/","montastic","MonTools","Moreover","Morfeus Fucking Scanner","Morning Paper","MovableType","mowser","Mrcgiguy","MS Web Services Client Protocol","MSFrontPage","mShots","MuckRack\\\/","muhstik-scan","MVAClient","MxToolbox\\\/","nagios","Najdi\\.si","Name Intelligence","Nameprotect","Navroad","NearSite","Needle","Nessus","Net Vampire","NetAnts","NETCRAFT","NetLyzer","NetMechanic","NetNewsWire","Netpursual","netresearch","NetShelter ContentScan","Netsparker","NetTrack","Netvibes","NetZIP","Neustar WPM","NeutrinoAPI","NewRelicPinger","NewsBlur .*Finder","NewsGator","newsme","newspaper\\\/","Nexgate Ruby Client","NG-Search","Nibbler","NICErsPRO","Nikto","nineconnections","NLNZ_IAHarvester","Nmap Scripting Engine","node-superagent","node-urllib","node\\.io","Nodemeter","NodePing","nominet\\.org\\.uk","nominet\\.uk","Norton-Safeweb","Notifixious","notifyninja","nuhk","nutch","Nuzzel","nWormFeedFinder","nyawc\\\/","Nymesis","NYU","Ocelli\\\/","Octopus","oegp","Offline Explorer","Offline Navigator","og-scraper","okhttp","omgili","OMSC","Online Domain Tools","OpenCalaisSemanticProxy","Openfind","OpenLinkProfiler","Openstat\\\/","OpenVAS","Optimizer","Orbiter","OrgProbe\\\/","orion-semantics","Outlook-Express","Outlook-iOS","ow\\.ly","Owler","ownCloud News","OxfordCloudService","Page Valet","page_verifier","page scorer","page2rss","PageGrabber","PagePeeker","PageScorer","Pagespeed\\\/","Panopta","panscient","Papa Foto","parsijoo","Pavuk","PayPal IPN","pcBrowser","Pcore-HTTP","Pearltrees","PECL::HTTP","peerindex","Peew","PeoplePal","Perlu -","PhantomJS Screenshoter","PhantomJS\\\/","Photon\\\/","phpservermon","Pi-Monster","Picscout","Picsearch","PictureFinder","Pimonster","ping\\.blo\\.gs","Pingability","PingAdmin\\.Ru","Pingdom","Pingoscope","PingSpot","pinterest\\.com","Pixray","Pizilla","Plagger\\\/","Ploetz \\+ Zeller","Plukkie","plumanalytics","PocketImageCache","PocketParser","Pockey","POE-Component-Client-HTTP","Polymail\\\/","Pompos","Porkbun","Port Monitor","postano","PostmanRuntime","PostPost","postrank","PowerPoint\\\/","Priceonomics Analysis Engine","PrintFriendly","PritTorrent","Prlog","probethenet","Project 25499","prospectb2b","Protopage","ProWebWalker","proximic","PRTG Network Monitor","pshtt, https scanning","PTST ","PTST\\\/[0-9]+","Pulsepoint XT3 web scraper","Pump","Python-httplib2","python-requests","Python-urllib","Qirina Hurdler","QQDownload","QrafterPro","Qseero","Qualidator","QueryN Metasearch","queuedriver","Quora Link Preview","Qwantify","Radian6","RankActive","RankFlex","RankSonicSiteAuditor","Re-re Studio","ReactorNetty","Readability","RealDownload","RealPlayer%20Downloader","RebelMouse","Recorder","RecurPost\\\/","redback\\\/","ReederForMac","ReGet","RepoMonkey","request\\.js","reqwest\\\/","ResponseCodeTest","RestSharp","Riddler","Rival IQ","Robosourcer","Robozilla","ROI Hunter","RPT-HTTPClient","RSSOwl","safe-agent-scanner","SalesIntelligent","Saleslift","Sendsay\\.Ru","SauceNAO","SBIder","scalaj-http","scan\\.lol","ScanAlert","Scoop","scooter","ScoutJet","ScoutURLMonitor","ScrapeBox Page Scanner","SimpleScraper","Scrapy","Screaming","ScreenShotService","Scrubby","Scrutiny\\\/","search\\.thunderstone","Search37","searchenginepromotionhelp","Searchestate","SearchExpress","SearchSight","Seeker","semanticdiscovery","semanticjuice","Semiocast HTTP client","Semrush","sentry\\\/","SEO Browser","Seo Servis","seo-nastroj\\.cz","seo4ajax","Seobility","SEOCentro","SeoCheck","SEOkicks","Seomoz","SEOprofiler","SEOsearch","seoscanners","seositecheckup","SEOstats","servernfo","sexsearcher","Seznam","Shelob","Shodan","Shoppimon","ShopWiki","ShortLinkTranslate","shrinktheweb","Sideqik","SimplePie","SimplyFast","Siphon","SISTRIX","Site-Shot\\\/","Site Sucker","Site24x7","SiteBar","Sitebeam","Sitebulb\\\/","SiteCondor","SiteExplorer","SiteGuardian","Siteimprove","SiteIndexed","Sitemap(s)? Generator","SitemapGenerator","SiteMonitor","Siteshooter B0t","SiteSnagger","SiteSucker","SiteTruth","Sitevigil","sitexy\\.com","SkypeUriPreview","Slack\\\/","slider\\.com","slurp","SlySearch","SmartDownload","SMRF URL Expander","SMUrlExpander","Snake","Snappy","SnapSearch","Snarfer\\\/","SniffRSS","sniptracker","Snoopy","SnowHaze Search","sogou web","SortSite","Sottopop","sovereign\\.ai","SpaceBison","SpamExperts","Spammen","Spanner","spaziodati","SPDYCheck","Specificfeeds","speedy","SPEng","Spinn3r","spray-can","Sprinklr ","spyonweb","sqlmap","Sqlworm","Sqworm","SSL Labs","ssl-tools","StackRambler","Statastico\\\/","StatusCake","Steeler","Stratagems Kumo","Stroke\\.cz","StudioFACA","StumbleUpon","suchen","Sucuri","summify","SuperHTTP","Surphace Scout","Suzuran","SwiteScraper","Symfony BrowserKit","Symfony2 BrowserKit","SynHttpClient-Built","Sysomos","sysscan","Szukacz","T0PHackTeam","tAkeOut","Tarantula\\\/","Taringa UGC","TarmotGezgin","Teleport","Telesoft","Telesphoreo","Telesphorep","Tenon\\.io","teoma","terrainformatica","Test Certificate Info","testuri","Tetrahedron","The Drop Reaper","The Expert HTML Source Viewer","The Knowledge AI","The Intraformant","theinternetrules","TheNomad","Thinklab","Thumbshots","ThumbSniper","timewe\\.net","TinEye","Tiny Tiny RSS","TLSProbe\\\/","Toata","topster","touche\\.com","Traackr\\.com","tracemyfile","Trackuity","TrapitAgent","Trendiction","Trendsmap","trendspottr","truwoGPS","TryJsoup","TulipChain","Turingos","Turnitin","tweetedtimes","Tweetminster","Tweezler\\\/","twibble","Twice","Twikle","Twingly","Twisted PageGetter","Typhoeus","ubermetrics-technologies","uclassify","UdmSearch","unchaos","unirest-java","UniversalFeedParser","Unshorten\\.It","Untiny","UnwindFetchor","updated","updown\\.io daemon","Upflow","Uptimia","Urlcheckr","URL Verifier","URLitor","urlresolver","Urlstat","URLTester","UrlTrends Ranking Updater","URLy Warning","URLy\\.Warning","Vacuum","Vagabondo","VB Project","vBSEO","VCI","via ggpht\\.com GoogleImageProxy","VidibleScraper","Virusdie","visionutils","vkShare","VoidEYE","Voil","voltron","voyager\\\/","VSAgent\\\/","VSB-TUO\\\/","Vulnbusters Meter","VYU2","w3af\\.org","W3C_Unicorn","W3C-checklink","W3C-mobileOK","WAC-OFU","Wallpapers\\\/[0-9]+","WallpapersHD","wangling","Wappalyzer","WatchMouse","WbSrch\\\/","WDT\\.io","web-capture\\.net","Web-sniffer","Web Auto","Web Collage","Web Enhancer","Web Fetch","Web Fuck","Web Pix","Web Sauger","Web Sucker","Webalta","Webauskunft","WebAuto","WebCapture","WebClient\\\/","webcollage","WebCookies","WebCopier","WebCorp","WebDataStats","WebDoc","WebEnhancer","WebFetch","WebFuck","WebGazer","WebGo IS","WebImageCollector","WebImages","WebIndex","webkit2png","WebLeacher","webmastercoffee","webmon ","WebPix","WebReaper","WebSauger","webscreenie","Webshag","Webshot","Website Quester","websitepulse agent","WebsiteQuester","Websnapr","WebSniffer","Webster","WebStripper","WebSucker","Webthumb\\\/","WebThumbnail","WebWhacker","WebZIP","WeLikeLinks","WEPA","WeSEE","wf84","Wfuzz\\\/","wget","WhatsApp","WhatsMyIP","WhatWeb","WhereGoes\\?","Whibse","WhoRunsCoinHive","Whynder Magnet","Windows-RSS-Platform","WinPodder","wkhtmlto","wmtips","Woko","woorankreview","Word\\\/","WordPress\\\/","WordupinfoSearch","wotbox","WP Engine Install Performance API","wpif","wprecon\\.com survey","WPScan","wscheck","Wtrace","WWW-Collector-E","WWW-Mechanize","WWW::Document","WWW::Mechanize","www\\.monitor\\.us","WWWOFFLE","x09Mozilla","x22Mozilla","XaxisSemanticsClassifier","Xenu Link Sleuth","XING-contenttabreceiver","xpymep([0-9]?)\\.exe","Y!J-(ASR|BSC)","Y\\!J-BRW","Yaanb","yacy","Yahoo Link Preview","YahooCacheSystem","YahooYSMcm","YandeG","Yandex(?!Search)","yanga","yeti","Yo-yo","Yoleo Consumer","yoogliFetchAgent","YottaaMonitor","Your-Website-Sucks","yourls\\.org","YoYs\\.net","YP\\.PL","Zabbix","Zade","Zao","Zauba","Zemanta Aggregator","Zend_Http_Client","Zend\\\\Http\\\\Client","Zermelo","Zeus ","zgrab","ZnajdzFoto","Zombie\\.js","Zoom\\.Mac","ZyBorg","[a-z0-9\\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer)"]
\ No newline at end of file
+[" YLT","^Aether","^Amazon Simple Notification Service Agent$","^Amazon-Route53-Health-Check-Service","^b0t$","^bluefish ","^Calypso v\\\/","^COMODO DCV","^Corax","^DangDang","^DavClnt","^DHSH","^docker\\\/[0-9]","^Expanse","^FDM ","^git\\\/","^Goose\\\/","^Grabber","^Gradle\\\/","^HTTPClient\\\/","^HTTPing","^Java\\\/","^Jeode\\\/","^Jetty\\\/","^Mail\\\/","^Mget","^Microsoft URL Control","^Mikrotik\\\/","^Netlab360","^NG\\\/[0-9\\.]","^NING\\\/","^npm\\\/","^Nuclei","^PHP-AYMAPI\\\/","^PHP\\\/","^pip\\\/","^pnpm\\\/","^RMA\\\/","^Ruby|Ruby\\\/[0-9]","^Swurl ","^TLS tester ","^twine\\\/","^ureq","^VSE\\\/[0-9]","^WordPress\\.com","^XRL\\\/[0-9]","^ZmEu","008\\\/","13TABS","192\\.comAgent","2GDPR\\\/","2ip\\.ru","404enemy","7Siters","80legs","a3logics\\.in","A6-Indexer","Abonti","Aboundex","aboutthedomain","Accoona-AI-Agent","acebookexternalhit\\\/","acoon","acrylicapps\\.com\\\/pulp","Acunetix","AdAuth\\\/","adbeat","AddThis","ADmantX","AdminLabs","adressendeutschland","adreview\\\/","adscanner","adstxt-worker","Adstxtaggregator","adstxt\\.com","Adyen HttpClient","AffiliateLabz\\\/","affilimate-puppeteer","agentslug","AHC","aihit","aiohttp\\\/","Airmail","akka-http\\\/","akula\\\/","alertra","alexa site audit","Alibaba\\.Security\\.Heimdall","Alligator","allloadin","AllSubmitter","alyze\\.info","amagit","Anarchie","AndroidDownloadManager","Anemone","AngleSharp","annotate_google","Anthill","Anturis Agent","Ant\\.com","AnyEvent-HTTP\\\/","Apache Ant\\\/","Apache Droid","Apache OpenOffice","Apache-HttpAsyncClient","Apache-HttpClient","ApacheBench","Apexoo","apimon\\.de","APIs-Google","AportWorm\\\/","AppBeat\\\/","AppEngine-Google","AppleSyndication","Aprc\\\/[0-9]","Arachmo","arachnode","Arachnophilia","aria2","Arukereso","asafaweb","Asana\\\/","Ask Jeeves","AskQuickly","ASPSeek","Asterias","Astute","asynchttp","Attach","attohttpc","autocite","AutomaticWPTester","Autonomy","awin\\.com","AWS Security Scanner","axios\\\/","a\\.pr-cy\\.ru","B-l-i-t-z-B-O-T","Backlink-Ceck","backlink-check","BacklinkHttpStatus","BackStreet","BackupLand","BackWeb","Bad-Neighborhood","Badass","baidu\\.com","Bandit","basicstate","BatchFTP","Battleztar Bazinga","baypup\\\/","BazQux","BBBike","BCKLINKS","BDFetch","BegunAdvertising","Bewica-security-scan","Bidtellect","BigBozz","Bigfoot","biglotron","BingLocalSearch","BingPreview","binlar","biNu image cacher","Bitacle","Bitrix link preview","biz_Directory","BKCTwitterUnshortener\\\/","Black Hole","Blackboard Safeassign","BlackWidow","BlockNote\\.Net","BlogBridge","Bloglines","Bloglovin","BlogPulseLive","BlogSearch","Blogtrottr","BlowFish","boitho\\.com-dc","Boost\\.Beast","BPImageWalker","Braintree-Webhooks","Branch Metrics API","Branch-Passthrough","Brandprotect","BrandVerity","Brandwatch","Brodie\\\/","Browsershots","BUbiNG","Buck\\\/","Buddy","BuiltWith","Bullseye","BunnySlippers","Burf Search","Butterfly\\\/","BuzzSumo","CAAM\\\/[0-9]","CakePHP","Calculon","Canary%20Mail","CaretNail","catexplorador","CC Metadata Scaper","Cegbfeieh","censys","centuryb.o.t9[at]gmail.com","Cerberian Drtrs","CERT\\.at-Statistics-Survey","cf-facebook","cg-eye","changedetection","ChangesMeter","Charlotte","chatterino-api-cache","CheckHost","checkprivacy","CherryPicker","ChinaClaw","Chirp\\\/","chkme\\.com","Chlooe","Chromaxa","CirrusExplorer","CISPA Vulnerability Notification","CISPA Web Analyser","Citoid","CJNetworkQuality","Clarsentia","clips\\.ua\\.ac\\.be","Cloud mapping","CloudEndure","CloudFlare-AlwaysOnline","Cloudflare-Healthchecks","Cloudinary","cmcm\\.com","coccoc","cognitiveseo","ColdFusion","colly -","CommaFeed","Commons-HttpClient","commonscan","contactbigdatafr","contentkingapp","Contextual Code Sites Explorer","convera","CookieReports","copyright sheriff","CopyRightCheck","Copyscape","cortex\\\/","Cosmos4j\\.feedback","Covario-IDS","Craw\\\/","Crescent","Criteo","Crowsnest","CSHttp","CSSCheck","Cula\\\/","curb","Curious George","curl","cuwhois\\\/","cybo\\.com","DAP\\\/NetHTTP","DareBoost","DatabaseDriverMysqli","DataCha0s","DatadogSynthetics","Datafeedwatch","Datanyze","DataparkSearch","dataprovider","DataXu","Daum(oa)?[ \\\/][0-9]","dBpoweramp","ddline","deeris","delve\\.ai","Demon","DeuSu","developers\\.google\\.com\\\/\\+\\\/web\\\/snippet\\\/","Devil","Digg","Digincore","DigitalPebble","Dirbuster","Discourse Forum Onebox","Dispatch\\\/","Disqus\\\/","DittoSpyder","dlvr","DMBrowser","DNSPod-reporting","docoloc","Dolphin http client","DomainAppender","DomainLabz","Domains Project\\\/","Donuts Content Explorer","dotMailer content retrieval","dotSemantic","downforeveryoneorjustme","Download Wonder","downnotifier","DowntimeDetector","Drip","drupact","Drupal \\(\\+http:\\\/\\\/drupal\\.org\\\/\\)","DTS Agent","dubaiindex","DuplexWeb-Google","DynatraceSynthetic","EARTHCOM","Easy-Thumb","EasyDL","Ebingbong","ec2linkfinder","eCairn-Grabber","eCatch","ECCP","eContext\\\/","Ecxi","EirGrabber","ElectricMonk","elefent","EMail Exractor","EMail Wolf","EmailWolf","Embarcadero","Embed PHP Library","Embedly","endo\\\/","europarchive\\.org","evc-batch","EventMachine HttpClient","Everwall Link Expander","Evidon","Evrinid","ExactSearch","ExaleadCloudview","Excel\\\/","exif","ExoRank","Exploratodo","Express WebPictures","Extreme Picture Finder","EyeNetIE","ezooms","facebookexternalhit","facebookexternalua","facebookplatform","fairshare","Faraday v","fasthttp","Faveeo","Favicon downloader","faviconarchive","faviconkit","FavOrg","Feed Wrangler","Feedable\\\/","Feedbin","FeedBooster","FeedBucket","FeedBunch\\\/","FeedBurner","feeder","Feedly","FeedshowOnline","Feedshow\\\/","Feedspot","FeedViewer\\\/","Feedwind\\\/","FeedZcollector","feeltiptop","Fetch API","Fetch\\\/[0-9]","Fever\\\/[0-9]","FHscan","Fiery%20Feeds","Filestack","Fimap","findlink","findthatfile","FlashGet","FlipboardBrowserProxy","FlipboardProxy","FlipboardRSS","Flock\\\/","Florienzh\\\/","fluffy","Flunky","flynxapp","forensiq","ForusP","FoundSeoTool","fragFINN\\.de","free thumbnails","Freeuploader","FreshRSS","frontman","Funnelback","Fuzz Faster U Fool","G-i-g-a-b-o-t","g00g1e\\.net","ganarvisitas","gdnplus\\.com","geek-tools","Genieo","GentleSource","GetCode","Getintent","GetLinkInfo","getprismatic","GetRight","getroot","GetURLInfo\\\/","GetWeb","Geziyor","Ghost Inspector","GigablastOpenSource","GIS-LABS","github-camo","GitHub-Hookshot","github\\.com","Go http package","Go [\\d\\.]* package http","Go!Zilla","Go-Ahead-Got-It","Go-http-client","go-mtasts\\\/","gobuster","gobyus","Gofeed","gofetch","Goldfire Server","GomezAgent","gooblog","Goodzer\\\/","Google AppsViewer","Google Desktop","Google favicon","Google Keyword Suggestion","Google Keyword Tool","Google Page Speed Insights","Google PP Default","Google Search Console","Google Web Preview","Google-Ads-Creatives-Assistant","Google-Ads-Overview","Google-Adwords","Google-Apps-Script","Google-Calendar-Importer","Google-HotelAdsVerifier","Google-HTTP-Java-Client","Google-InspectionTool","Google-Podcast","Google-Publisher-Plugin","Google-Read-Aloud","Google-SearchByImage","Google-Site-Verification","Google-SMTP-STS","Google-speakr","Google-Structured-Data-Testing-Tool","Google-Transparency-Report","google-xrawler","Google-Youtube-Links","GoogleDocs","GoogleHC\\\/","GoogleProber","GoogleProducer","GoogleSites","Gookey","GoSpotCheck","gosquared-thumbnailer","Gotit","GoZilla","grabify","GrabNet","Grafula","Grammarly","GrapeFX","GreatNews","Gregarius","GRequests","grokkit","grouphigh","grub-client","gSOAP\\\/","GT::WWW","GTmetrix","GuzzleHttp","gvfs\\\/","HAA(A)?RTLAND http client","Haansoft","hackney\\\/","Hadi Agent","HappyApps-WebCheck","Hardenize","Hatena","Havij","HaxerMen","HeadlessChrome","HEADMasterSEO","HeartRails_Capture","help@dataminr\\.com","heritrix","Hexometer","historious","hkedcity","hledejLevne\\.cz","Hloader","HMView","Holmes","HonesoSearchEngine","HootSuite Image proxy","Hootsuite-WebFeed","hosterstats","HostTracker","ht:\\\/\\\/check","htdig","HTMLparser","htmlyse","HTTP Banner Detection","http-get","HTTP-Header-Abfrage","http-kit","http-request\\\/","HTTP-Tiny","HTTP::Lite","http:\\\/\\\/www.neomo.de\\\/","HttpComponents","httphr","HTTPie","HTTPMon","httpRequest","httpscheck","httpssites_power","httpunit","HttpUrlConnection","http\\.rb\\\/","HTTP_Compression_Test","http_get","http_request2","http_requester","httrack","huaweisymantec","HubSpot ","HubSpot-Link-Resolver","Humanlinks","i2kconnect\\\/","Iblog","ichiro","Id-search","IdeelaborPlagiaat","IDG Twitter Links Resolver","IDwhois\\\/","Iframely","igdeSpyder","iGooglePortal","IlTrovatore","Image Fetch","Image Sucker","ImageEngine\\\/","ImageVisu\\\/","Imagga","imagineeasy","imgsizer","InAGist","inbound\\.li parser","InDesign%20CC","Indy Library","InetURL","infegy","infohelfer","InfoTekies","InfoWizards Reciprocal Link","inpwrd\\.com","instabid","Instapaper","Integrity","integromedb","Intelliseek","InterGET","Internet Ninja","InternetSeer","internetVista monitor","internetwache","internet_archive","intraVnews","IODC","IOI","Inboxb0t","iplabel","ips-agent","IPS\\\/[0-9]","IPWorks HTTP\\\/S Component","iqdb\\\/","Iria","Irokez","isitup\\.org","iskanie","isUp\\.li","iThemes Sync\\\/","IZaBEE","iZSearch","JAHHO","janforman","Jaunt\\\/","Java.*outbrain","javelin\\.io","Jbrofuzz","Jersey\\\/","JetCar","Jigsaw","Jobboerse","JobFeed discovery","Jobg8 URL Monitor","jobo","Jobrapido","Jobsearch1\\.5","JoinVision Generic","JolokiaPwn","Joomla","Jorgee","JS-Kit","JungleKeyThumbnail","JustView","Kaspersky Lab CFR link resolver","Kelny\\\/","Kerrigan\\\/","KeyCDN","Keyword Density","Keywords Research","khttp\\\/","KickFire","KimonoLabs\\\/","Kml-Google","knows\\.is","KOCMOHABT","kouio","kube-probe","kubectl","kulturarw3","KumKie","Larbin","Lavf\\\/","leakix\\.net","LeechFTP","LeechGet","letsencrypt","Lftp","LibVLC","LibWeb","Libwhisker","libwww","Licorne","Liferea\\\/","Lighthouse","Lightspeedsystems","Likse","limber\\.io","Link Valet","LinkAlarm\\\/","LinkAnalyser","linkCheck","linkdex","LinkExaminer","linkfluence","linkpeek","LinkPreview","LinkScan","LinksManager","LinkTiger","LinkWalker","link_thumbnailer","Lipperhey","Litemage_walker","livedoor ScreenShot","LoadImpactRload","localsearch-web","LongURL API","longurl-r-package","looid\\.com","looksystems\\.net","ltx71","lua-resty-http","Lucee \\(CFML Engine\\)","Lush Http Client","lwp-request","lwp-trivial","LWP::Simple","lycos","LYT\\.SR","L\\.webis","mabontland","MacOutlook\\\/","Mag-Net","MagpieRSS","Mail::STS","MailChimp","Mail\\.Ru","Majestic12","makecontact\\\/","Mandrill","MapperCmd","marketinggrader","MarkMonitor","MarkWatch","Mass Downloader","masscan\\\/","Mata Hari","mattermost","Mediametric","Mediapartners-Google","mediawords","MegaIndex\\.ru","MeltwaterNews","Melvil Rawi","MemGator","Metaspinner","MetaURI","MFC_Tear_Sample","Microsearch","Microsoft Data Access","Microsoft Office","Microsoft Outlook","Microsoft Windows Network Diagnostics","Microsoft-WebDAV-MiniRedir","Microsoft\\.Data\\.Mashup","MIDown tool","MIIxpc","Mindjet","Miniature\\.io","Miniflux","mio_httpc","Miro-HttpClient","Mister PiX","mixdata dot com","mixed-content-scan","mixnode","Mnogosearch","mogimogi","Mojeek","Mojolicious \\(Perl\\)","Mollie","monitis","Monitority\\\/","Monit\\\/","montastic","MonTools","Moreover","Morfeus Fucking Scanner","Morning Paper","MovableType","mowser","Mrcgiguy","Mr\\.4x3 Powered","MS Web Services Client Protocol","MSFrontPage","mShots","MuckRack\\\/","muhstik-scan","MVAClient","MxToolbox\\\/","myseosnapshot","nagios","Najdi\\.si","Name Intelligence","NameFo\\.com","Nameprotect","nationalarchives","Navroad","NearSite","Needle","Nessus","Net Vampire","NetAnts","NETCRAFT","NetLyzer","NetMechanic","NetNewsWire","Netpursual","netresearch","NetShelter ContentScan","Netsparker","NetSystemsResearch","nettle","NetTrack","Netvibes","NetZIP","Neustar WPM","NeutrinoAPI","NewRelicPinger","NewsBlur .*Finder","NewsGator","newsme","newspaper\\\/","Nexgate Ruby Client","NG-Search","nghttp2","Nibbler","NICErsPRO","NihilScio","Nikto","nineconnections","NLNZ_IAHarvester","Nmap Scripting Engine","node-fetch","node-superagent","node-urllib","Nodemeter","NodePing","node\\.io","nominet\\.org\\.uk","nominet\\.uk","Norton-Safeweb","Notifixious","notifyninja","NotionEmbedder","nuhk","nutch","Nuzzel","nWormFeedFinder","nyawc\\\/","Nymesis","NYU","Observatory\\\/","Ocelli\\\/","Octopus","oegp","Offline Explorer","Offline Navigator","OgScrper","okhttp","omgili","OMSC","Online Domain Tools","Open Source RSS","OpenCalaisSemanticProxy","Openfind","OpenLinkProfiler","Openstat\\\/","OpenVAS","OPPO A33","Optimizer","Orbiter","OrgProbe\\\/","orion-semantics","Outlook-Express","Outlook-iOS","Owler","Owlin","ownCloud News","ow\\.ly","OxfordCloudService","page scorer","Page Valet","page2rss","PageFreezer","PageGrabber","PagePeeker","PageScorer","Pagespeed\\\/","PageThing","page_verifier","Panopta","panscient","Papa Foto","parsijoo","Pavuk","PayPal IPN","pcBrowser","Pcore-HTTP","PDF24 URL To PDF","Pearltrees","PECL::HTTP","peerindex","Peew","PeoplePal","Perlu -","PhantomJS Screenshoter","PhantomJS\\\/","Photon\\\/","php-requests","phpservermon","Pi-Monster","Picscout","Picsearch","PictureFinder","Pimonster","Pingability","PingAdmin\\.Ru","Pingdom","Pingoscope","PingSpot","ping\\.blo\\.gs","pinterest\\.com","Pixray","Pizilla","Plagger\\\/","Pleroma ","Ploetz \\+ Zeller","Plukkie","plumanalytics","PocketImageCache","PocketParser","Pockey","PodcastAddict\\\/","POE-Component-Client-HTTP","Polymail\\\/","Pompos","Porkbun","Port Monitor","postano","postfix-mta-sts-resolver","PostmanRuntime","postplanner\\.com","PostPost","postrank","PowerPoint\\\/","Prebid","Prerender","Priceonomics Analysis Engine","PrintFriendly","PritTorrent","Prlog","probethenet","Project ?25499","Project-Resonance","prospectb2b","Protopage","ProWebWalker","proximic","PRTG Network Monitor","pshtt, https scanning","PTST ","PTST\\\/[0-9]+","Pump","Python-httplib2","python-httpx","python-requests","Python-urllib","Qirina Hurdler","QQDownload","QrafterPro","Qseero","Qualidator","QueryN Metasearch","queuedriver","quic-go-HTTP\\\/","QuiteRSS","Quora Link Preview","Qwantify","Radian6","RadioPublicImageResizer","Railgun\\\/","RankActive","RankFlex","RankSonicSiteAuditor","RapidLoad\\\/","Re-re Studio","ReactorNetty","Readability","RealDownload","RealPlayer%20Downloader","RebelMouse","Recorder","RecurPost\\\/","redback\\\/","ReederForMac","Reeder\\\/","ReGet","RepoMonkey","request\\.js","reqwest\\\/","ResponseCodeTest","RestSharp","Riddler","Rival IQ","Robosourcer","Robozilla","ROI Hunter","RPT-HTTPClient","RSSMix\\\/","RSSOwl","RyowlEngine","safe-agent-scanner","SalesIntelligent","Saleslift","SAP NetWeaver Application Server","SauceNAO","SBIder","sc-downloader","scalaj-http","Scamadviser-Frontend","ScanAlert","scan\\.lol","Scoop","scooter","ScopeContentAG-HTTP-Client","ScoutJet","ScoutURLMonitor","ScrapeBox Page Scanner","Scrapy","Screaming","ScreenShotService","Scrubby","Scrutiny\\\/","Search37","searchenginepromotionhelp","Searchestate","SearchExpress","SearchSight","SearchWP","search\\.thunderstone","Seeker","semanticdiscovery","semanticjuice","Semiocast HTTP client","Semrush","Sendsay\\.Ru","sentry\\\/","SEO Browser","Seo Servis","seo-nastroj\\.cz","seo4ajax","Seobility","SEOCentro","SeoCheck","seocompany","SEOkicks","SEOlizer","Seomoz","SEOprofiler","seoscanners","SEOsearch","seositecheckup","SEOstats","servernfo","sexsearcher","Seznam","Shelob","Shodan","Shoppimon","ShopWiki","ShortLinkTranslate","shortURL lengthener","shrinktheweb","Sideqik","Siege","SimplePie","SimplyFast","Siphon","SISTRIX","Site Sucker","Site-Shot\\\/","Site24x7","SiteBar","Sitebeam","Sitebulb\\\/","SiteCondor","SiteExplorer","SiteGuardian","Siteimprove","SiteIndexed","Sitemap(s)? Generator","SitemapGenerator","SiteMonitor","Siteshooter B0t","SiteSnagger","SiteSucker","SiteTruth","Sitevigil","sitexy\\.com","SkypeUriPreview","Slack\\\/","sli-systems\\.com","slider\\.com","slurp","SlySearch","SmartDownload","SMRF URL Expander","SMUrlExpander","Snake","Snappy","SnapSearch","Snarfer\\\/","SniffRSS","sniptracker","Snoopy","SnowHaze Search","sogou web","SortSite","Sottopop","sovereign\\.ai","SpaceBison","SpamExperts","Spammen","Spanner","Spawning-AI","spaziodati","SPDYCheck","Specificfeeds","SpeedKit","speedy","SPEng","Spinn3r","spray-can","Sprinklr ","spyonweb","sqlmap","Sqlworm","Sqworm","SSL Labs","ssl-tools","StackRambler","Statastico\\\/","Statically-","StatusCake","Steeler","Stratagems Kumo","Stripe\\\/","Stroke\\.cz","StudioFACA","StumbleUpon","suchen","Sucuri","summify","SuperHTTP","Surphace Scout","Suzuran","swcd ","Symfony BrowserKit","Symfony2 BrowserKit","Synapse\\\/","Syndirella\\\/","SynHttpClient-Built","Sysomos","sysscan","Szukacz","T0PHackTeam","tAkeOut","Tarantula\\\/","Taringa UGC","TarmotGezgin","tchelebi\\.io","techiaith\\.cymru","Teleport","Telesoft","Telesphoreo","Telesphorep","Tenon\\.io","teoma","terrainformatica","Test Certificate Info","testuri","Tetrahedron","TextRazor Downloader","The Drop Reaper","The Expert HTML Source Viewer","The Intraformant","The Knowledge AI","theinternetrules","TheNomad","Thinklab","Thumbor","Thumbshots","ThumbSniper","timewe\\.net","TinEye","Tiny Tiny RSS","TLSProbe\\\/","Toata","topster","touche\\.com","Traackr\\.com","tracemyfile","Trackuity","TrapitAgent","Trendiction","Trendsmap","trendspottr","truwoGPS","TryJsoup","TulipChain","Turingos","Turnitin","tweetedtimes","Tweetminster","Tweezler\\\/","twibble","Twice","Twikle","Twingly","Twisted PageGetter","Typhoeus","ubermetrics-technologies","uclassify","UdmSearch","ultimate_sitemap_parser","unchaos","unirest-java","UniversalFeedParser","unshortenit","Unshorten\\.It","Untiny","UnwindFetchor","updated","updown\\.io daemon","Upflow","Uptimia","URL Verifier","Urlcheckr","URLitor","urlresolver","Urlstat","URLTester","UrlTrends Ranking Updater","URLy Warning","URLy\\.Warning","URL\\\/Emacs","Vacuum","Vagabondo","VB Project","vBSEO","VCI","via ggpht\\.com GoogleImageProxy","Virusdie","visionutils","Visual Rights Group","vkShare","VoidEYE","Voil","voltron","voyager\\\/","VSAgent\\\/","VSB-TUO\\\/","Vulnbusters Meter","VYU2","w3af\\.org","W3C-checklink","W3C-mobileOK","W3C_Unicorn","WAC-OFU","WakeletLinkExpander","WallpapersHD","Wallpapers\\\/[0-9]+","wangling","Wappalyzer","WatchMouse","WbSrch\\\/","WDT\\.io","Web Auto","Web Collage","Web Enhancer","Web Fetch","Web Fuck","Web Pix","Web Sauger","Web spyder","Web Sucker","web-capture\\.net","Web-sniffer","Webalta","Webauskunft","WebAuto","WebCapture","WebClient\\\/","webcollage","WebCookies","WebCopier","WebCorp","WebDataStats","WebDoc","WebEnhancer","WebFetch","WebFuck","WebGazer","WebGo IS","WebImageCollector","WebImages","WebIndex","webkit2png","WebLeacher","webmastercoffee","webmon ","WebPix","WebReaper","WebSauger","webscreenie","Webshag","Webshot","Website Quester","websitepulse agent","WebsiteQuester","Websnapr","WebSniffer","Webster","WebStripper","WebSucker","webtech\\\/","WebThumbnail","Webthumb\\\/","WebWhacker","WebZIP","WeLikeLinks","WEPA","WeSEE","wf84","Wfuzz\\\/","wget","WhatCMS","WhatsApp","WhatsMyIP","WhatWeb","WhereGoes\\?","Whibse","WhoAPI\\\/","WhoRunsCoinHive","Whynder Magnet","Windows-RSS-Platform","WinHttp-Autoproxy-Service","WinHTTP\\\/","WinPodder","wkhtmlto","wmtips","Woko","Wolfram HTTPClient","woorankreview","WordPress\\\/","WordupinfoSearch","Word\\\/","worldping-api","wotbox","WP Engine Install Performance API","WP Rocket","wpif","wprecon\\.com survey","WPScan","wscheck","Wtrace","WWW-Collector-E","WWW-Mechanize","WWW::Document","WWW::Mechanize","WWWOFFLE","www\\.monitor\\.us","x09Mozilla","x22Mozilla","XaxisSemanticsClassifier","XenForo\\\/","Xenu Link Sleuth","XING-contenttabreceiver","xpymep([0-9]?)\\.exe","Y!J-[A-Z][A-Z][A-Z]","Yaanb","yacy","Yahoo Link Preview","YahooCacheSystem","YahooMailProxy","YahooYSMcm","YandeG","Yandex(?!Search)","yanga","yeti","Yo-yo","Yoleo Consumer","yomins\\.com","yoogliFetchAgent","YottaaMonitor","Your-Website-Sucks","yourls\\.org","YoYs\\.net","YP\\.PL","Zabbix","Zade","Zao","Zauba","Zemanta Aggregator","Zend\\\\Http\\\\Client","Zend_Http_Client","Zermelo","Zeus ","zgrab","ZnajdzFoto","ZnHTTP","Zombie\\.js","Zoom\\.Mac","ZoteroTranslationServer","ZyBorg","[a-z0-9\\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)"]
\ No newline at end of file
index 1522796e9471ebf1f7e46d830611d94b9f286ea6..791d08bb240eb72dedd66945c4777a5e56ad226e 100644 (file)
@@ -1,27 +1,46 @@
-.*Java.*outbrain
  YLT
+^Aether
+^Amazon Simple Notification Service Agent$
+^Amazon-Route53-Health-Check-Service
 ^b0t$
 ^bluefish 
 ^Calypso v\/
 ^COMODO DCV
+^Corax
 ^DangDang
 ^DavClnt
+^DHSH
+^docker\/[0-9]
+^Expanse
 ^FDM 
 ^git\/
 ^Goose\/
 ^Grabber
+^Gradle\/
 ^HTTPClient\/
+^HTTPing
 ^Java\/
 ^Jeode\/
 ^Jetty\/
 ^Mail\/
 ^Mget
 ^Microsoft URL Control
+^Mikrotik\/
+^Netlab360
 ^NG\/[0-9\.]
 ^NING\/
-^PHP\/[0-9]
+^npm\/
+^Nuclei
+^PHP-AYMAPI\/
+^PHP\/
+^pip\/
+^pnpm\/
 ^RMA\/
 ^Ruby|Ruby\/[0-9]
+^Swurl 
+^TLS tester 
+^twine\/
+^ureq
 ^VSE\/[0-9]
 ^WordPress\.com
 ^XRL\/[0-9]
 008\/
 13TABS
 192\.comAgent
+2GDPR\/
 2ip\.ru
 404enemy
 7Siters
 80legs
-a\.pr-cy\.ru
 a3logics\.in
 A6-Indexer
 Abonti
 Aboundex
 aboutthedomain
 Accoona-AI-Agent
+acebookexternalhit\/
 acoon
 acrylicapps\.com\/pulp
 Acunetix
@@ -49,8 +69,14 @@ AddThis
 ADmantX
 AdminLabs
 adressendeutschland
+adreview\/
 adscanner
+adstxt-worker
 Adstxtaggregator
+adstxt\.com
+Adyen HttpClient
+AffiliateLabz\/
+affilimate-puppeteer
 agentslug
 AHC
 aihit
@@ -71,20 +97,23 @@ AndroidDownloadManager
 Anemone
 AngleSharp
 annotate_google
-Ant\.com
+Anthill
 Anturis Agent
+Ant\.com
 AnyEvent-HTTP\/
+Apache Ant\/
 Apache Droid
 Apache OpenOffice
 Apache-HttpAsyncClient
 Apache-HttpClient
 ApacheBench
 Apexoo
+apimon\.de
 APIs-Google
 AportWorm\/
 AppBeat\/
 AppEngine-Google
-AppStoreScraperZ
+AppleSyndication
 Aprc\/[0-9]
 Arachmo
 arachnode
@@ -92,21 +121,28 @@ Arachnophilia
 aria2
 Arukereso
 asafaweb
-AskQuickly
+Asana\/
 Ask Jeeves
+AskQuickly
 ASPSeek
 Asterias
 Astute
 asynchttp
 Attach
+attohttpc
 autocite
+AutomaticWPTester
 Autonomy
+awin\.com
+AWS Security Scanner
 axios\/
+a\.pr-cy\.ru
 B-l-i-t-z-B-O-T
 Backlink-Ceck
 backlink-check
 BacklinkHttpStatus
 BackStreet
+BackupLand
 BackWeb
 Bad-Neighborhood
 Badass
@@ -121,6 +157,7 @@ BBBike
 BCKLINKS
 BDFetch
 BegunAdvertising
+Bewica-security-scan
 Bidtellect
 BigBozz
 Bigfoot
@@ -130,11 +167,14 @@ BingPreview
 binlar
 biNu image cacher
 Bitacle
+Bitrix link preview
 biz_Directory
+BKCTwitterUnshortener\/
 Black Hole
 Blackboard Safeassign
 BlackWidow
 BlockNote\.Net
+BlogBridge
 Bloglines
 Bloglovin
 BlogPulseLive
@@ -142,6 +182,7 @@ BlogSearch
 Blogtrottr
 BlowFish
 boitho\.com-dc
+Boost\.Beast
 BPImageWalker
 Braintree-Webhooks
 Branch Metrics API
@@ -169,12 +210,15 @@ catexplorador
 CC Metadata Scaper
 Cegbfeieh
 censys
+centuryb.o.t9[at]gmail.com
 Cerberian Drtrs
 CERT\.at-Statistics-Survey
+cf-facebook
 cg-eye
 changedetection
 ChangesMeter
 Charlotte
+chatterino-api-cache
 CheckHost
 checkprivacy
 CherryPicker
@@ -185,6 +229,7 @@ Chlooe
 Chromaxa
 CirrusExplorer
 CISPA Vulnerability Notification
+CISPA Web Analyser
 Citoid
 CJNetworkQuality
 Clarsentia
@@ -192,27 +237,34 @@ clips\.ua\.ac\.be
 Cloud mapping
 CloudEndure
 CloudFlare-AlwaysOnline
+Cloudflare-Healthchecks
 Cloudinary
 cmcm\.com
 coccoc
 cognitiveseo
+ColdFusion
 colly -
 CommaFeed
 Commons-HttpClient
 commonscan
 contactbigdatafr
 contentkingapp
+Contextual Code Sites Explorer
 convera
 CookieReports
 copyright sheriff
 CopyRightCheck
 Copyscape
+cortex\/
 Cosmos4j\.feedback
 Covario-IDS
+Craw\/
 Crescent
-Crowsnest
 Criteo
+Crowsnest
 CSHttp
+CSSCheck
+Cula\/
 curb
 Curious George
 curl
@@ -222,12 +274,17 @@ DAP\/NetHTTP
 DareBoost
 DatabaseDriverMysqli
 DataCha0s
+DatadogSynthetics
 Datafeedwatch
 Datanyze
 DataparkSearch
 dataprovider
 DataXu
 Daum(oa)?[ \/][0-9]
+dBpoweramp
+ddline
+deeris
+delve\.ai
 Demon
 DeuSu
 developers\.google\.com\/\+\/web\/snippet\/
@@ -237,8 +294,8 @@ Digincore
 DigitalPebble
 Dirbuster
 Discourse Forum Onebox
-Disqus\/
 Dispatch\/
+Disqus\/
 DittoSpyder
 dlvr
 DMBrowser
@@ -246,6 +303,8 @@ DNSPod-reporting
 docoloc
 Dolphin http client
 DomainAppender
+DomainLabz
+Domains Project\/
 Donuts Content Explorer
 dotMailer content retrieval
 dotSemantic
@@ -258,6 +317,8 @@ drupact
 Drupal \(\+http:\/\/drupal\.org\/\)
 DTS Agent
 dubaiindex
+DuplexWeb-Google
+DynatraceSynthetic
 EARTHCOM
 Easy-Thumb
 EasyDL
@@ -288,20 +349,22 @@ ExactSearch
 ExaleadCloudview
 Excel\/
 exif
+ExoRank
 Exploratodo
 Express WebPictures
 Extreme Picture Finder
 EyeNetIE
 ezooms
 facebookexternalhit
+facebookexternalua
 facebookplatform
 fairshare
 Faraday v
 fasthttp
 Faveeo
 Favicon downloader
-faviconkit
 faviconarchive
+faviconkit
 FavOrg
 Feed Wrangler
 Feedable\/
@@ -313,7 +376,9 @@ FeedBurner
 feeder
 Feedly
 FeedshowOnline
+Feedshow\/
 Feedspot
+FeedViewer\/
 Feedwind\/
 FeedZcollector
 feeltiptop
@@ -321,6 +386,8 @@ Fetch API
 Fetch\/[0-9]
 Fever\/[0-9]
 FHscan
+Fiery%20Feeds
+Filestack
 Fimap
 findlink
 findthatfile
@@ -329,18 +396,24 @@ FlipboardBrowserProxy
 FlipboardProxy
 FlipboardRSS
 Flock\/
+Florienzh\/
 fluffy
 Flunky
 flynxapp
 forensiq
+ForusP
 FoundSeoTool
-http:\/\/www.neomo.de\/
+fragFINN\.de
 free thumbnails
 Freeuploader
+FreshRSS
+frontman
 Funnelback
+Fuzz Faster U Fool
 G-i-g-a-b-o-t
 g00g1e\.net
 ganarvisitas
+gdnplus\.com
 geek-tools
 Genieo
 GentleSource
@@ -352,18 +425,24 @@ GetRight
 getroot
 GetURLInfo\/
 GetWeb
+Geziyor
 Ghost Inspector
 GigablastOpenSource
 GIS-LABS
 github-camo
+GitHub-Hookshot
 github\.com
-Go [\d\.]* package http
 Go http package
+Go [\d\.]* package http
+Go!Zilla
 Go-Ahead-Got-It
 Go-http-client
-Go!Zilla
+go-mtasts\/
+gobuster
 gobyus
+Gofeed
 gofetch
+Goldfire Server
 GomezAgent
 gooblog
 Goodzer\/
@@ -376,24 +455,31 @@ Google Page Speed Insights
 Google PP Default
 Google Search Console
 Google Web Preview
+Google-Ads-Creatives-Assistant
+Google-Ads-Overview
 Google-Adwords
 Google-Apps-Script
 Google-Calendar-Importer
 Google-HotelAdsVerifier
 Google-HTTP-Java-Client
+Google-InspectionTool
+Google-Podcast
 Google-Publisher-Plugin
+Google-Read-Aloud
 Google-SearchByImage
 Google-Site-Verification
+Google-SMTP-STS
+Google-speakr
 Google-Structured-Data-Testing-Tool
-Google-Youtube-Links
+Google-Transparency-Report
 google-xrawler
+Google-Youtube-Links
 GoogleDocs
 GoogleHC\/
+GoogleProber
 GoogleProducer
 GoogleSites
-Google-Transparency-Report
 Gookey
-GoScraper
 GoSpotCheck
 gosquared-thumbnailer
 Gotit
@@ -419,13 +505,16 @@ Haansoft
 hackney\/
 Hadi Agent
 HappyApps-WebCheck
+Hardenize
 Hatena
 Havij
+HaxerMen
 HeadlessChrome
 HEADMasterSEO
 HeartRails_Capture
 help@dataminr\.com
 heritrix
+Hexometer
 historious
 hkedcity
 hledejLevne\.cz
@@ -442,28 +531,31 @@ htdig
 HTMLparser
 htmlyse
 HTTP Banner Detection
-HTTP_Compression_Test
-http_request2
-http_requester
 http-get
 HTTP-Header-Abfrage
 http-kit
 http-request\/
 HTTP-Tiny
 HTTP::Lite
-http\.rb\/
-http_get
+http:\/\/www.neomo.de\/
 HttpComponents
 httphr
+HTTPie
 HTTPMon
 httpRequest
 httpscheck
 httpssites_power
 httpunit
 HttpUrlConnection
+http\.rb\/
+HTTP_Compression_Test
+http_get
+http_request2
+http_requester
 httrack
 huaweisymantec
 HubSpot 
+HubSpot-Link-Resolver
 Humanlinks
 i2kconnect\/
 Iblog
@@ -474,6 +566,7 @@ IDG Twitter Links Resolver
 IDwhois\/
 Iframely
 igdeSpyder
+iGooglePortal
 IlTrovatore
 Image Fetch
 Image Sucker
@@ -498,13 +591,15 @@ Integrity
 integromedb
 Intelliseek
 InterGET
-internet_archive
 Internet Ninja
 InternetSeer
 internetVista monitor
+internetwache
+internet_archive
 intraVnews
 IODC
 IOI
+Inboxb0t
 iplabel
 ips-agent
 IPS\/[0-9]
@@ -516,10 +611,13 @@ isitup\.org
 iskanie
 isUp\.li
 iThemes Sync\/
+IZaBEE
 iZSearch
 JAHHO
 janforman
 Jaunt\/
+Java.*outbrain
+javelin\.io
 Jbrofuzz
 Jersey\/
 JetCar
@@ -535,6 +633,7 @@ JolokiaPwn
 Joomla
 Jorgee
 JS-Kit
+JungleKeyThumbnail
 JustView
 Kaspersky Lab CFR link resolver
 Kelny\/
@@ -542,6 +641,7 @@ Kerrigan\/
 KeyCDN
 Keyword Density
 Keywords Research
+khttp\/
 KickFire
 KimonoLabs\/
 Kml-Google
@@ -549,11 +649,12 @@ knows\.is
 KOCMOHABT
 kouio
 kube-probe
+kubectl
 kulturarw3
 KumKie
-L\.webis
 Larbin
 Lavf\/
+leakix\.net
 LeechFTP
 LeechGet
 letsencrypt
@@ -564,41 +665,50 @@ Libwhisker
 libwww
 Licorne
 Liferea\/
-Lightspeedsystems
 Lighthouse
+Lightspeedsystems
 Likse
+limber\.io
 Link Valet
-link_thumbnailer
 LinkAlarm\/
+LinkAnalyser
 linkCheck
 linkdex
 LinkExaminer
 linkfluence
 linkpeek
-LinkPreviewGenerator
+LinkPreview
 LinkScan
 LinksManager
 LinkTiger
 LinkWalker
+link_thumbnailer
 Lipperhey
 Litemage_walker
 livedoor ScreenShot
 LoadImpactRload
 localsearch-web
 LongURL API
+longurl-r-package
+looid\.com
 looksystems\.net
 ltx71
 lua-resty-http
+Lucee \(CFML Engine\)
+Lush Http Client
 lwp-request
 lwp-trivial
 LWP::Simple
 lycos
 LYT\.SR
+L\.webis
 mabontland
+MacOutlook\/
 Mag-Net
 MagpieRSS
-Mail\.Ru
+Mail::STS
 MailChimp
+Mail\.Ru
 Majestic12
 makecontact\/
 Mandrill
@@ -609,6 +719,8 @@ MarkWatch
 Mass Downloader
 masscan\/
 Mata Hari
+mattermost
+Mediametric
 Mediapartners-Google
 mediawords
 MegaIndex\.ru
@@ -619,28 +731,31 @@ Metaspinner
 MetaURI
 MFC_Tear_Sample
 Microsearch
-Microsoft Office 
+Microsoft Data Access
+Microsoft Office
 Microsoft Outlook
 Microsoft Windows Network Diagnostics
 Microsoft-WebDAV-MiniRedir
-Microsoft Data Access
+Microsoft\.Data\.Mashup
 MIDown tool
 MIIxpc
 Mindjet
 Miniature\.io
 Miniflux
+mio_httpc
+Miro-HttpClient
 Mister PiX
 mixdata dot com
 mixed-content-scan
-Mixmax-LinkPreview
 mixnode
 Mnogosearch
 mogimogi
 Mojeek
 Mojolicious \(Perl\)
-Monit\/
+Mollie
 monitis
 Monitority\/
+Monit\/
 montastic
 MonTools
 Moreover
@@ -649,6 +764,7 @@ Morning Paper
 MovableType
 mowser
 Mrcgiguy
+Mr\.4x3 Powered
 MS Web Services Client Protocol
 MSFrontPage
 mShots
@@ -656,10 +772,13 @@ MuckRack\/
 muhstik-scan
 MVAClient
 MxToolbox\/
+myseosnapshot
 nagios
 Najdi\.si
 Name Intelligence
+NameFo\.com
 Nameprotect
+nationalarchives
 Navroad
 NearSite
 Needle
@@ -674,6 +793,8 @@ Netpursual
 netresearch
 NetShelter ContentScan
 Netsparker
+NetSystemsResearch
+nettle
 NetTrack
 Netvibes
 NetZIP
@@ -686,22 +807,26 @@ newsme
 newspaper\/
 Nexgate Ruby Client
 NG-Search
+nghttp2
 Nibbler
 NICErsPRO
+NihilScio
 Nikto
 nineconnections
 NLNZ_IAHarvester
 Nmap Scripting Engine
+node-fetch
 node-superagent
 node-urllib
-node\.io
 Nodemeter
 NodePing
+node\.io
 nominet\.org\.uk
 nominet\.uk
 Norton-Safeweb
 Notifixious
 notifyninja
+NotionEmbedder
 nuhk
 nutch
 Nuzzel
@@ -709,39 +834,45 @@ nWormFeedFinder
 nyawc\/
 Nymesis
 NYU
+Observatory\/
 Ocelli\/
 Octopus
 oegp
 Offline Explorer
 Offline Navigator
-og-scraper
+OgScrper
 okhttp
 omgili
 OMSC
 Online Domain Tools
+Open Source RSS
 OpenCalaisSemanticProxy
 Openfind
 OpenLinkProfiler
 Openstat\/
 OpenVAS
+OPPO A33
 Optimizer
 Orbiter
 OrgProbe\/
 orion-semantics
 Outlook-Express
 Outlook-iOS
-ow\.ly
 Owler
+Owlin
 ownCloud News
+ow\.ly
 OxfordCloudService
-Page Valet
-page_verifier
 page scorer
+Page Valet
 page2rss
+PageFreezer
 PageGrabber
 PagePeeker
 PageScorer
 Pagespeed\/
+PageThing
+page_verifier
 Panopta
 panscient
 Papa Foto
@@ -750,6 +881,7 @@ Pavuk
 PayPal IPN
 pcBrowser
 Pcore-HTTP
+PDF24 URL To PDF
 Pearltrees
 PECL::HTTP
 peerindex
@@ -759,44 +891,52 @@ Perlu -
 PhantomJS Screenshoter
 PhantomJS\/
 Photon\/
+php-requests
 phpservermon
 Pi-Monster
 Picscout
 Picsearch
 PictureFinder
 Pimonster
-ping\.blo\.gs
 Pingability
 PingAdmin\.Ru
 Pingdom
 Pingoscope
 PingSpot
+ping\.blo\.gs
 pinterest\.com
 Pixray
 Pizilla
 Plagger\/
+Pleroma 
 Ploetz \+ Zeller
 Plukkie
 plumanalytics
 PocketImageCache
 PocketParser
 Pockey
+PodcastAddict\/
 POE-Component-Client-HTTP
 Polymail\/
 Pompos
 Porkbun
 Port Monitor
 postano
+postfix-mta-sts-resolver
 PostmanRuntime
+postplanner\.com
 PostPost
 postrank
 PowerPoint\/
+Prebid
+Prerender
 Priceonomics Analysis Engine
 PrintFriendly
 PritTorrent
 Prlog
 probethenet
-Project 25499
+Project ?25499
+Project-Resonance
 prospectb2b
 Protopage
 ProWebWalker
@@ -805,9 +945,9 @@ PRTG Network Monitor
 pshtt, https scanning
 PTST 
 PTST\/[0-9]+
-Pulsepoint XT3 web scraper
 Pump
 Python-httplib2
+python-httpx
 python-requests
 Python-urllib
 Qirina Hurdler
@@ -817,12 +957,17 @@ Qseero
 Qualidator
 QueryN Metasearch
 queuedriver
+quic-go-HTTP\/
+QuiteRSS
 Quora Link Preview
 Qwantify
 Radian6
+RadioPublicImageResizer
+Railgun\/
 RankActive
 RankFlex
 RankSonicSiteAuditor
+RapidLoad\/
 Re-re Studio
 ReactorNetty
 Readability
@@ -833,6 +978,7 @@ Recorder
 RecurPost\/
 redback\/
 ReederForMac
+Reeder\/
 ReGet
 RepoMonkey
 request\.js
@@ -845,38 +991,44 @@ Robosourcer
 Robozilla
 ROI Hunter
 RPT-HTTPClient
+RSSMix\/
 RSSOwl
+RyowlEngine
 safe-agent-scanner
 SalesIntelligent
 Saleslift
-Sendsay\.Ru
+SAP NetWeaver Application Server
 SauceNAO
 SBIder
+sc-downloader
 scalaj-http
-scan\.lol
+Scamadviser-Frontend
 ScanAlert
+scan\.lol
 Scoop
 scooter
+ScopeContentAG-HTTP-Client
 ScoutJet
 ScoutURLMonitor
 ScrapeBox Page Scanner
-SimpleScraper
 Scrapy
 Screaming
 ScreenShotService
 Scrubby
 Scrutiny\/
-search\.thunderstone
 Search37
 searchenginepromotionhelp
 Searchestate
 SearchExpress
 SearchSight
+SearchWP
+search\.thunderstone
 Seeker
 semanticdiscovery
 semanticjuice
 Semiocast HTTP client
 Semrush
+Sendsay\.Ru
 sentry\/
 SEO Browser
 Seo Servis
@@ -885,11 +1037,13 @@ seo4ajax
 Seobility
 SEOCentro
 SeoCheck
+seocompany
 SEOkicks
+SEOlizer
 Seomoz
 SEOprofiler
-SEOsearch
 seoscanners
+SEOsearch
 seositecheckup
 SEOstats
 servernfo
@@ -900,14 +1054,16 @@ Shodan
 Shoppimon
 ShopWiki
 ShortLinkTranslate
+shortURL lengthener
 shrinktheweb
 Sideqik
+Siege
 SimplePie
 SimplyFast
 Siphon
 SISTRIX
-Site-Shot\/
 Site Sucker
+Site-Shot\/
 Site24x7
 SiteBar
 Sitebeam
@@ -928,6 +1084,7 @@ Sitevigil
 sitexy\.com
 SkypeUriPreview
 Slack\/
+sli-systems\.com
 slider\.com
 slurp
 SlySearch
@@ -950,9 +1107,11 @@ SpaceBison
 SpamExperts
 Spammen
 Spanner
+Spawning-AI
 spaziodati
 SPDYCheck
 Specificfeeds
+SpeedKit
 speedy
 SPEng
 Spinn3r
@@ -966,9 +1125,11 @@ SSL Labs
 ssl-tools
 StackRambler
 Statastico\/
+Statically-
 StatusCake
 Steeler
 Stratagems Kumo
+Stripe\/
 Stroke\.cz
 StudioFACA
 StumbleUpon
@@ -978,9 +1139,11 @@ summify
 SuperHTTP
 Surphace Scout
 Suzuran
-SwiteScraper
+swcd 
 Symfony BrowserKit
 Symfony2 BrowserKit
+Synapse\/
+Syndirella\/
 SynHttpClient-Built
 Sysomos
 sysscan
@@ -990,6 +1153,8 @@ tAkeOut
 Tarantula\/
 Taringa UGC
 TarmotGezgin
+tchelebi\.io
+techiaith\.cymru
 Teleport
 Telesoft
 Telesphoreo
@@ -1000,13 +1165,15 @@ terrainformatica
 Test Certificate Info
 testuri
 Tetrahedron
+TextRazor Downloader
 The Drop Reaper
 The Expert HTML Source Viewer
-The Knowledge AI
 The Intraformant
+The Knowledge AI
 theinternetrules
 TheNomad
 Thinklab
+Thumbor
 Thumbshots
 ThumbSniper
 timewe\.net
@@ -1040,9 +1207,11 @@ Typhoeus
 ubermetrics-technologies
 uclassify
 UdmSearch
+ultimate_sitemap_parser
 unchaos
 unirest-java
 UniversalFeedParser
+unshortenit
 Unshorten\.It
 Untiny
 UnwindFetchor
@@ -1050,8 +1219,8 @@ updated
 updown\.io daemon
 Upflow
 Uptimia
-Urlcheckr
 URL Verifier
+Urlcheckr
 URLitor
 urlresolver
 Urlstat
@@ -1059,15 +1228,16 @@ URLTester
 UrlTrends Ranking Updater
 URLy Warning
 URLy\.Warning
+URL\/Emacs
 Vacuum
 Vagabondo
 VB Project
 vBSEO
 VCI
 via ggpht\.com GoogleImageProxy
-VidibleScraper
 Virusdie
 visionutils
+Visual Rights Group
 vkShare
 VoidEYE
 Voil
@@ -1078,19 +1248,18 @@ VSB-TUO\/
 Vulnbusters Meter
 VYU2
 w3af\.org
-W3C_Unicorn
 W3C-checklink
 W3C-mobileOK
+W3C_Unicorn
 WAC-OFU
-Wallpapers\/[0-9]+
+WakeletLinkExpander
 WallpapersHD
+Wallpapers\/[0-9]+
 wangling
 Wappalyzer
 WatchMouse
 WbSrch\/
 WDT\.io
-web-capture\.net
-Web-sniffer
 Web Auto
 Web Collage
 Web Enhancer
@@ -1098,7 +1267,10 @@ Web Fetch
 Web Fuck
 Web Pix
 Web Sauger
+Web spyder
 Web Sucker
+web-capture\.net
+Web-sniffer
 Webalta
 Webauskunft
 WebAuto
@@ -1136,8 +1308,9 @@ WebSniffer
 Webster
 WebStripper
 WebSucker
-Webthumb\/
+webtech\/
 WebThumbnail
+Webthumb\/
 WebWhacker
 WebZIP
 WeLikeLinks
@@ -1146,24 +1319,31 @@ WeSEE
 wf84
 Wfuzz\/
 wget
+WhatCMS
 WhatsApp
 WhatsMyIP
 WhatWeb
 WhereGoes\?
 Whibse
+WhoAPI\/
 WhoRunsCoinHive
 Whynder Magnet
 Windows-RSS-Platform
+WinHttp-Autoproxy-Service
+WinHTTP\/
 WinPodder
 wkhtmlto
 wmtips
 Woko
+Wolfram HTTPClient
 woorankreview
-Word\/
 WordPress\/
 WordupinfoSearch
+Word\/
+worldping-api
 wotbox
 WP Engine Install Performance API
+WP Rocket
 wpif
 wprecon\.com survey
 WPScan
@@ -1173,20 +1353,21 @@ WWW-Collector-E
 WWW-Mechanize
 WWW::Document
 WWW::Mechanize
-www\.monitor\.us
 WWWOFFLE
+www\.monitor\.us
 x09Mozilla
 x22Mozilla
 XaxisSemanticsClassifier
+XenForo\/
 Xenu Link Sleuth
 XING-contenttabreceiver
 xpymep([0-9]?)\.exe
-Y!J-(ASR|BSC)
-Y\!J-BRW
+Y!J-[A-Z][A-Z][A-Z]
 Yaanb
 yacy
 Yahoo Link Preview
 YahooCacheSystem
+YahooMailProxy
 YahooYSMcm
 YandeG
 Yandex(?!Search)
@@ -1194,6 +1375,7 @@ yanga
 yeti
 Yo-yo
 Yoleo Consumer
+yomins\.com
 yoogliFetchAgent
 YottaaMonitor
 Your-Website-Sucks
@@ -1205,13 +1387,15 @@ Zade
 Zao
 Zauba
 Zemanta Aggregator
-Zend_Http_Client
 Zend\\Http\\Client
+Zend_Http_Client
 Zermelo
 Zeus 
 zgrab
 ZnajdzFoto
+ZnHTTP
 Zombie\.js
 Zoom\.Mac
+ZoteroTranslationServer
 ZyBorg
-[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer)
\ No newline at end of file
+[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)
\ No newline at end of file
index a18eb985ed773a6e4414900add53e72384e1b53e..e7e014167111b465972e0af7d7ac038f71342a56 100644 (file)
@@ -1 +1 @@
-["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT",";"]
\ No newline at end of file
+["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT","OCTOPUS-CORE","htc_botdugls","super\\\/\\d+\\\/Android\\\/\\d+"]
\ No newline at end of file
index da56db9bd086d89c6138fa96295645dd47a58dee..a44a99cb07fd3ef1be80a8d1cf6789558d8cc2e3 100644 (file)
@@ -45,4 +45,6 @@ cubot
 ; IDbot
 ; ID bot
 ; POWER BOT
-;
\ No newline at end of file
+OCTOPUS-CORE
+htc_botdugls
+super\/\d+\/Android\/\d+
\ No newline at end of file
index 1067976be348a0c8877456d807c72b2c70d854dd..3ea284a7df210020573223481c47dc945b9a5409 100644 (file)
@@ -20,9 +20,9 @@ class CrawlerDetect
     /**
      * The user agent.
      *
-     * @var null
+     * @var string|null
      */
-    protected $userAgent = null;
+    protected $userAgent;
 
     /**
      * Headers that contain a user agent.
@@ -93,7 +93,7 @@ class CrawlerDetect
      * Compile the regex patterns into one regex string.
      *
      * @param array
-     * 
+     *
      * @return string
      */
     public function compileRegex($patterns)
@@ -138,7 +138,7 @@ class CrawlerDetect
     /**
      * Set the user agent.
      *
-     * @param string $userAgent
+     * @param string|null $userAgent
      */
     public function setUserAgent($userAgent)
     {
@@ -165,20 +165,14 @@ class CrawlerDetect
         $agent = trim(preg_replace(
             "/{$this->compiledExclusions}/i",
             '',
-            $userAgent ?: $this->userAgent
+            $userAgent ?: $this->userAgent ?: ''
         ));
 
-        if ($agent == '') {
+        if ($agent === '') {
             return false;
         }
 
-        $result = preg_match("/{$this->compiledRegex}/i", $agent, $matches);
-
-        if ($matches) {
-            $this->matches = $matches;
-        }
-
-        return (bool) $result;
+        return (bool) preg_match("/{$this->compiledRegex}/i", $agent, $this->matches);
     }
 
     /**
@@ -190,4 +184,13 @@ class CrawlerDetect
     {
         return isset($this->matches[0]) ? $this->matches[0] : null;
     }
+
+
+    /**
+     * @return string|null
+     */
+    public function getUserAgent()
+    {
+        return $this->userAgent;
+    }
 }
index 26ea8e5fc44b4dec61b44cb904a2f72f92b1da83..ffe10f51920e78ede2e318f957c110dbcc78797d 100644 (file)
@@ -15,14 +15,14 @@ abstract class AbstractProvider
 {
     /**
      * The data set.
-     * 
+     *
      * @var array
      */
     protected $data;
 
     /**
      * Return the data set.
-     * 
+     *
      * @return array
      */
     public function getAll()
index a907056580c6d61d2356a2640383390a48deb24d..a44edd23e45cfa043ae0ce21c14761eed0eea106 100644 (file)
@@ -19,30 +19,49 @@ class Crawlers extends AbstractProvider
      * @var array
      */
     protected $data = array(
-        '.*Java.*outbrain',
         ' YLT',
+        '^Aether',
+        '^Amazon Simple Notification Service Agent$',
+        '^Amazon-Route53-Health-Check-Service',
         '^b0t$',
         '^bluefish ',
         '^Calypso v\/',
         '^COMODO DCV',
+        '^Corax',
         '^DangDang',
         '^DavClnt',
+        '^DHSH',
+        '^docker\/[0-9]',
+        '^Expanse',
         '^FDM ',
         '^git\/',
         '^Goose\/',
         '^Grabber',
+        '^Gradle\/',
         '^HTTPClient\/',
+        '^HTTPing',
         '^Java\/',
         '^Jeode\/',
         '^Jetty\/',
         '^Mail\/',
         '^Mget',
         '^Microsoft URL Control',
+        '^Mikrotik\/',
+        '^Netlab360',
         '^NG\/[0-9\.]',
         '^NING\/',
-        '^PHP\/[0-9]',
+        '^npm\/',
+        '^Nuclei',
+        '^PHP-AYMAPI\/',
+        '^PHP\/',
+        '^pip\/',
+        '^pnpm\/',
         '^RMA\/',
         '^Ruby|Ruby\/[0-9]',
+        '^Swurl ',
+        '^TLS tester ',
+        '^twine\/',
+        '^ureq',
         '^VSE\/[0-9]',
         '^WordPress\.com',
         '^XRL\/[0-9]',
@@ -50,17 +69,18 @@ class Crawlers extends AbstractProvider
         '008\/',
         '13TABS',
         '192\.comAgent',
+        '2GDPR\/',
         '2ip\.ru',
         '404enemy',
         '7Siters',
         '80legs',
-        'a\.pr-cy\.ru',
         'a3logics\.in',
         'A6-Indexer',
         'Abonti',
         'Aboundex',
         'aboutthedomain',
         'Accoona-AI-Agent',
+        'acebookexternalhit\/',
         'acoon',
         'acrylicapps\.com\/pulp',
         'Acunetix',
@@ -70,8 +90,14 @@ class Crawlers extends AbstractProvider
         'ADmantX',
         'AdminLabs',
         'adressendeutschland',
+        'adreview\/',
         'adscanner',
+        'adstxt-worker',
         'Adstxtaggregator',
+        'adstxt\.com',
+        'Adyen HttpClient',
+        'AffiliateLabz\/',
+        'affilimate-puppeteer',
         'agentslug',
         'AHC',
         'aihit',
@@ -92,20 +118,23 @@ class Crawlers extends AbstractProvider
         'Anemone',
         'AngleSharp',
         'annotate_google',
-        'Ant\.com',
+        'Anthill',
         'Anturis Agent',
+        'Ant\.com',
         'AnyEvent-HTTP\/',
+        'Apache Ant\/',
         'Apache Droid',
         'Apache OpenOffice',
         'Apache-HttpAsyncClient',
         'Apache-HttpClient',
         'ApacheBench',
         'Apexoo',
+        'apimon\.de',
         'APIs-Google',
         'AportWorm\/',
         'AppBeat\/',
         'AppEngine-Google',
-        'AppStoreScraperZ',
+        'AppleSyndication',
         'Aprc\/[0-9]',
         'Arachmo',
         'arachnode',
@@ -113,21 +142,28 @@ class Crawlers extends AbstractProvider
         'aria2',
         'Arukereso',
         'asafaweb',
-        'AskQuickly',
+        'Asana\/',
         'Ask Jeeves',
+        'AskQuickly',
         'ASPSeek',
         'Asterias',
         'Astute',
         'asynchttp',
         'Attach',
+        'attohttpc',
         'autocite',
+        'AutomaticWPTester',
         'Autonomy',
+        'awin\.com',
+        'AWS Security Scanner',
         'axios\/',
+        'a\.pr-cy\.ru',
         'B-l-i-t-z-B-O-T',
         'Backlink-Ceck',
         'backlink-check',
         'BacklinkHttpStatus',
         'BackStreet',
+        'BackupLand',
         'BackWeb',
         'Bad-Neighborhood',
         'Badass',
@@ -142,6 +178,7 @@ class Crawlers extends AbstractProvider
         'BCKLINKS',
         'BDFetch',
         'BegunAdvertising',
+        'Bewica-security-scan',
         'Bidtellect',
         'BigBozz',
         'Bigfoot',
@@ -151,11 +188,14 @@ class Crawlers extends AbstractProvider
         'binlar',
         'biNu image cacher',
         'Bitacle',
+        'Bitrix link preview',
         'biz_Directory',
+        'BKCTwitterUnshortener\/',
         'Black Hole',
         'Blackboard Safeassign',
         'BlackWidow',
         'BlockNote\.Net',
+        'BlogBridge',
         'Bloglines',
         'Bloglovin',
         'BlogPulseLive',
@@ -163,6 +203,7 @@ class Crawlers extends AbstractProvider
         'Blogtrottr',
         'BlowFish',
         'boitho\.com-dc',
+        'Boost\.Beast',
         'BPImageWalker',
         'Braintree-Webhooks',
         'Branch Metrics API',
@@ -190,12 +231,15 @@ class Crawlers extends AbstractProvider
         'CC Metadata Scaper',
         'Cegbfeieh',
         'censys',
+        'centuryb.o.t9[at]gmail.com',
         'Cerberian Drtrs',
         'CERT\.at-Statistics-Survey',
+        'cf-facebook',
         'cg-eye',
         'changedetection',
         'ChangesMeter',
         'Charlotte',
+        'chatterino-api-cache',
         'CheckHost',
         'checkprivacy',
         'CherryPicker',
@@ -206,6 +250,7 @@ class Crawlers extends AbstractProvider
         'Chromaxa',
         'CirrusExplorer',
         'CISPA Vulnerability Notification',
+        'CISPA Web Analyser',
         'Citoid',
         'CJNetworkQuality',
         'Clarsentia',
@@ -213,27 +258,34 @@ class Crawlers extends AbstractProvider
         'Cloud mapping',
         'CloudEndure',
         'CloudFlare-AlwaysOnline',
+        'Cloudflare-Healthchecks',
         'Cloudinary',
         'cmcm\.com',
         'coccoc',
         'cognitiveseo',
+        'ColdFusion',
         'colly -',
         'CommaFeed',
         'Commons-HttpClient',
         'commonscan',
         'contactbigdatafr',
         'contentkingapp',
+        'Contextual Code Sites Explorer',
         'convera',
         'CookieReports',
         'copyright sheriff',
         'CopyRightCheck',
         'Copyscape',
+        'cortex\/',
         'Cosmos4j\.feedback',
         'Covario-IDS',
+        'Craw\/',
         'Crescent',
-        'Crowsnest',
         'Criteo',
+        'Crowsnest',
         'CSHttp',
+        'CSSCheck',
+        'Cula\/',
         'curb',
         'Curious George',
         'curl',
@@ -243,12 +295,17 @@ class Crawlers extends AbstractProvider
         'DareBoost',
         'DatabaseDriverMysqli',
         'DataCha0s',
+        'DatadogSynthetics',
         'Datafeedwatch',
         'Datanyze',
         'DataparkSearch',
         'dataprovider',
         'DataXu',
         'Daum(oa)?[ \/][0-9]',
+        'dBpoweramp',
+        'ddline',
+        'deeris',
+        'delve\.ai',
         'Demon',
         'DeuSu',
         'developers\.google\.com\/\+\/web\/snippet\/',
@@ -258,8 +315,8 @@ class Crawlers extends AbstractProvider
         'DigitalPebble',
         'Dirbuster',
         'Discourse Forum Onebox',
-        'Disqus\/',
         'Dispatch\/',
+        'Disqus\/',
         'DittoSpyder',
         'dlvr',
         'DMBrowser',
@@ -267,6 +324,8 @@ class Crawlers extends AbstractProvider
         'docoloc',
         'Dolphin http client',
         'DomainAppender',
+        'DomainLabz',
+        'Domains Project\/',
         'Donuts Content Explorer',
         'dotMailer content retrieval',
         'dotSemantic',
@@ -279,6 +338,8 @@ class Crawlers extends AbstractProvider
         'Drupal \(\+http:\/\/drupal\.org\/\)',
         'DTS Agent',
         'dubaiindex',
+        'DuplexWeb-Google',
+        'DynatraceSynthetic',
         'EARTHCOM',
         'Easy-Thumb',
         'EasyDL',
@@ -309,20 +370,22 @@ class Crawlers extends AbstractProvider
         'ExaleadCloudview',
         'Excel\/',
         'exif',
+        'ExoRank',
         'Exploratodo',
         'Express WebPictures',
         'Extreme Picture Finder',
         'EyeNetIE',
         'ezooms',
         'facebookexternalhit',
+        'facebookexternalua',
         'facebookplatform',
         'fairshare',
         'Faraday v',
         'fasthttp',
         'Faveeo',
         'Favicon downloader',
-        'faviconkit',
         'faviconarchive',
+        'faviconkit',
         'FavOrg',
         'Feed Wrangler',
         'Feedable\/',
@@ -334,7 +397,9 @@ class Crawlers extends AbstractProvider
         'feeder',
         'Feedly',
         'FeedshowOnline',
+        'Feedshow\/',
         'Feedspot',
+        'FeedViewer\/',
         'Feedwind\/',
         'FeedZcollector',
         'feeltiptop',
@@ -342,6 +407,8 @@ class Crawlers extends AbstractProvider
         'Fetch\/[0-9]',
         'Fever\/[0-9]',
         'FHscan',
+        'Fiery%20Feeds',
+        'Filestack',
         'Fimap',
         'findlink',
         'findthatfile',
@@ -350,18 +417,24 @@ class Crawlers extends AbstractProvider
         'FlipboardProxy',
         'FlipboardRSS',
         'Flock\/',
+        'Florienzh\/',
         'fluffy',
         'Flunky',
         'flynxapp',
         'forensiq',
+        'ForusP',
         'FoundSeoTool',
-        'http:\/\/www.neomo.de\/', //'Francis [Bot]'
+        'fragFINN\.de',
         'free thumbnails',
         'Freeuploader',
+        'FreshRSS',
+        'frontman',
         'Funnelback',
+        'Fuzz Faster U Fool',
         'G-i-g-a-b-o-t',
         'g00g1e\.net',
         'ganarvisitas',
+        'gdnplus\.com',
         'geek-tools',
         'Genieo',
         'GentleSource',
@@ -373,18 +446,24 @@ class Crawlers extends AbstractProvider
         'getroot',
         'GetURLInfo\/',
         'GetWeb',
+        'Geziyor',
         'Ghost Inspector',
         'GigablastOpenSource',
         'GIS-LABS',
         'github-camo',
+        'GitHub-Hookshot',
         'github\.com',
-        'Go [\d\.]* package http',
         'Go http package',
+        'Go [\d\.]* package http',
+        'Go!Zilla',
         'Go-Ahead-Got-It',
         'Go-http-client',
-        'Go!Zilla',
+        'go-mtasts\/',
+        'gobuster',
         'gobyus',
+        'Gofeed',
         'gofetch',
+        'Goldfire Server',
         'GomezAgent',
         'gooblog',
         'Goodzer\/',
@@ -397,24 +476,31 @@ class Crawlers extends AbstractProvider
         'Google PP Default',
         'Google Search Console',
         'Google Web Preview',
+        'Google-Ads-Creatives-Assistant',
+        'Google-Ads-Overview',
         'Google-Adwords',
         'Google-Apps-Script',
         'Google-Calendar-Importer',
         'Google-HotelAdsVerifier',
         'Google-HTTP-Java-Client',
+        'Google-InspectionTool',
+        'Google-Podcast',
         'Google-Publisher-Plugin',
+        'Google-Read-Aloud',
         'Google-SearchByImage',
         'Google-Site-Verification',
+        'Google-SMTP-STS',
+        'Google-speakr',
         'Google-Structured-Data-Testing-Tool',
-        'Google-Youtube-Links',
+        'Google-Transparency-Report',
         'google-xrawler',
+        'Google-Youtube-Links',
         'GoogleDocs',
         'GoogleHC\/',
+        'GoogleProber',
         'GoogleProducer',
         'GoogleSites',
-        'Google-Transparency-Report',
         'Gookey',
-        'GoScraper',
         'GoSpotCheck',
         'gosquared-thumbnailer',
         'Gotit',
@@ -440,13 +526,16 @@ class Crawlers extends AbstractProvider
         'hackney\/',
         'Hadi Agent',
         'HappyApps-WebCheck',
+        'Hardenize',
         'Hatena',
         'Havij',
+        'HaxerMen',
         'HeadlessChrome',
         'HEADMasterSEO',
         'HeartRails_Capture',
         'help@dataminr\.com',
         'heritrix',
+        'Hexometer',
         'historious',
         'hkedcity',
         'hledejLevne\.cz',
@@ -463,28 +552,31 @@ class Crawlers extends AbstractProvider
         'HTMLparser',
         'htmlyse',
         'HTTP Banner Detection',
-        'HTTP_Compression_Test',
-        'http_request2',
-        'http_requester',
         'http-get',
         'HTTP-Header-Abfrage',
         'http-kit',
         'http-request\/',
         'HTTP-Tiny',
         'HTTP::Lite',
-        'http\.rb\/',
-        'http_get',
+        'http:\/\/www.neomo.de\/', //'Francis [Bot]'
         'HttpComponents',
         'httphr',
+        'HTTPie',
         'HTTPMon',
         'httpRequest',
         'httpscheck',
         'httpssites_power',
         'httpunit',
         'HttpUrlConnection',
+        'http\.rb\/',
+        'HTTP_Compression_Test',
+        'http_get',
+        'http_request2',
+        'http_requester',
         'httrack',
         'huaweisymantec',
         'HubSpot ',
+        'HubSpot-Link-Resolver',
         'Humanlinks',
         'i2kconnect\/',
         'Iblog',
@@ -495,6 +587,7 @@ class Crawlers extends AbstractProvider
         'IDwhois\/',
         'Iframely',
         'igdeSpyder',
+        'iGooglePortal',
         'IlTrovatore',
         'Image Fetch',
         'Image Sucker',
@@ -519,13 +612,15 @@ class Crawlers extends AbstractProvider
         'integromedb',
         'Intelliseek',
         'InterGET',
-        'internet_archive',
         'Internet Ninja',
         'InternetSeer',
         'internetVista monitor',
+        'internetwache',
+        'internet_archive',
         'intraVnews',
         'IODC',
         'IOI',
+        'Inboxb0t',
         'iplabel',
         'ips-agent',
         'IPS\/[0-9]',
@@ -537,10 +632,13 @@ class Crawlers extends AbstractProvider
         'iskanie',
         'isUp\.li',
         'iThemes Sync\/',
+        'IZaBEE',
         'iZSearch',
         'JAHHO',
         'janforman',
         'Jaunt\/',
+        'Java.*outbrain',
+        'javelin\.io',
         'Jbrofuzz',
         'Jersey\/',
         'JetCar',
@@ -556,6 +654,7 @@ class Crawlers extends AbstractProvider
         'Joomla',
         'Jorgee',
         'JS-Kit',
+        'JungleKeyThumbnail',
         'JustView',
         'Kaspersky Lab CFR link resolver',
         'Kelny\/',
@@ -563,6 +662,7 @@ class Crawlers extends AbstractProvider
         'KeyCDN',
         'Keyword Density',
         'Keywords Research',
+        'khttp\/',
         'KickFire',
         'KimonoLabs\/',
         'Kml-Google',
@@ -570,11 +670,12 @@ class Crawlers extends AbstractProvider
         'KOCMOHABT',
         'kouio',
         'kube-probe',
+        'kubectl',
         'kulturarw3',
         'KumKie',
-        'L\.webis',
         'Larbin',
         'Lavf\/',
+        'leakix\.net',
         'LeechFTP',
         'LeechGet',
         'letsencrypt',
@@ -585,41 +686,50 @@ class Crawlers extends AbstractProvider
         'libwww',
         'Licorne',
         'Liferea\/',
-        'Lightspeedsystems',
         'Lighthouse',
+        'Lightspeedsystems',
         'Likse',
+        'limber\.io',
         'Link Valet',
-        'link_thumbnailer',
         'LinkAlarm\/',
+        'LinkAnalyser',
         'linkCheck',
         'linkdex',
         'LinkExaminer',
         'linkfluence',
         'linkpeek',
-        'LinkPreviewGenerator',
+        'LinkPreview',
         'LinkScan',
         'LinksManager',
         'LinkTiger',
         'LinkWalker',
+        'link_thumbnailer',
         'Lipperhey',
         'Litemage_walker',
         'livedoor ScreenShot',
         'LoadImpactRload',
         'localsearch-web',
         'LongURL API',
+        'longurl-r-package',
+        'looid\.com',
         'looksystems\.net',
         'ltx71',
         'lua-resty-http',
+        'Lucee \(CFML Engine\)',
+        'Lush Http Client',
         'lwp-request',
         'lwp-trivial',
         'LWP::Simple',
         'lycos',
         'LYT\.SR',
+        'L\.webis',
         'mabontland',
+        'MacOutlook\/',
         'Mag-Net',
         'MagpieRSS',
-        'Mail\.Ru',
+        'Mail::STS',
         'MailChimp',
+        'Mail\.Ru',
         'Majestic12',
         'makecontact\/',
         'Mandrill',
@@ -630,6 +740,8 @@ class Crawlers extends AbstractProvider
         'Mass Downloader',
         'masscan\/',
         'Mata Hari',
+        'mattermost',
+        'Mediametric',
         'Mediapartners-Google',
         'mediawords',
         'MegaIndex\.ru',
@@ -640,28 +752,31 @@ class Crawlers extends AbstractProvider
         'MetaURI',
         'MFC_Tear_Sample',
         'Microsearch',
-        'Microsoft Office ',
+        'Microsoft Data Access',
+        'Microsoft Office',
         'Microsoft Outlook',
         'Microsoft Windows Network Diagnostics',
         'Microsoft-WebDAV-MiniRedir',
-        'Microsoft Data Access',
+        'Microsoft\.Data\.Mashup',
         'MIDown tool',
         'MIIxpc',
         'Mindjet',
         'Miniature\.io',
         'Miniflux',
+        'mio_httpc',
+        'Miro-HttpClient',
         'Mister PiX',
         'mixdata dot com',
         'mixed-content-scan',
-        'Mixmax-LinkPreview',
         'mixnode',
         'Mnogosearch',
         'mogimogi',
         'Mojeek',
         'Mojolicious \(Perl\)',
-        'Monit\/',
+        'Mollie',
         'monitis',
         'Monitority\/',
+        'Monit\/',
         'montastic',
         'MonTools',
         'Moreover',
@@ -670,6 +785,7 @@ class Crawlers extends AbstractProvider
         'MovableType',
         'mowser',
         'Mrcgiguy',
+        'Mr\.4x3 Powered',
         'MS Web Services Client Protocol',
         'MSFrontPage',
         'mShots',
@@ -677,10 +793,13 @@ class Crawlers extends AbstractProvider
         'muhstik-scan',
         'MVAClient',
         'MxToolbox\/',
+        'myseosnapshot',
         'nagios',
         'Najdi\.si',
         'Name Intelligence',
+        'NameFo\.com',
         'Nameprotect',
+        'nationalarchives',
         'Navroad',
         'NearSite',
         'Needle',
@@ -695,6 +814,8 @@ class Crawlers extends AbstractProvider
         'netresearch',
         'NetShelter ContentScan',
         'Netsparker',
+        'NetSystemsResearch',
+        'nettle',
         'NetTrack',
         'Netvibes',
         'NetZIP',
@@ -707,22 +828,26 @@ class Crawlers extends AbstractProvider
         'newspaper\/',
         'Nexgate Ruby Client',
         'NG-Search',
+        'nghttp2',
         'Nibbler',
         'NICErsPRO',
+        'NihilScio',
         'Nikto',
         'nineconnections',
         'NLNZ_IAHarvester',
         'Nmap Scripting Engine',
+        'node-fetch',
         'node-superagent',
         'node-urllib',
-        'node\.io',
         'Nodemeter',
         'NodePing',
+        'node\.io',
         'nominet\.org\.uk',
         'nominet\.uk',
         'Norton-Safeweb',
         'Notifixious',
         'notifyninja',
+        'NotionEmbedder',
         'nuhk',
         'nutch',
         'Nuzzel',
@@ -730,39 +855,45 @@ class Crawlers extends AbstractProvider
         'nyawc\/',
         'Nymesis',
         'NYU',
+        'Observatory\/',
         'Ocelli\/',
         'Octopus',
         'oegp',
         'Offline Explorer',
         'Offline Navigator',
-        'og-scraper',
+        'OgScrper',
         'okhttp',
         'omgili',
         'OMSC',
         'Online Domain Tools',
+        'Open Source RSS',
         'OpenCalaisSemanticProxy',
         'Openfind',
         'OpenLinkProfiler',
         'Openstat\/',
         'OpenVAS',
+        'OPPO A33',
         'Optimizer',
         'Orbiter',
         'OrgProbe\/',
         'orion-semantics',
         'Outlook-Express',
         'Outlook-iOS',
-        'ow\.ly',
         'Owler',
+        'Owlin',
         'ownCloud News',
+        'ow\.ly',
         'OxfordCloudService',
-        'Page Valet',
-        'page_verifier',
         'page scorer',
+        'Page Valet',
         'page2rss',
+        'PageFreezer',
         'PageGrabber',
         'PagePeeker',
         'PageScorer',
         'Pagespeed\/',
+        'PageThing',
+        'page_verifier',
         'Panopta',
         'panscient',
         'Papa Foto',
@@ -771,6 +902,7 @@ class Crawlers extends AbstractProvider
         'PayPal IPN',
         'pcBrowser',
         'Pcore-HTTP',
+        'PDF24 URL To PDF',
         'Pearltrees',
         'PECL::HTTP',
         'peerindex',
@@ -780,44 +912,52 @@ class Crawlers extends AbstractProvider
         'PhantomJS Screenshoter',
         'PhantomJS\/',
         'Photon\/',
+        'php-requests',
         'phpservermon',
         'Pi-Monster',
         'Picscout',
         'Picsearch',
         'PictureFinder',
         'Pimonster',
-        'ping\.blo\.gs',
         'Pingability',
         'PingAdmin\.Ru',
         'Pingdom',
         'Pingoscope',
         'PingSpot',
+        'ping\.blo\.gs',
         'pinterest\.com',
         'Pixray',
         'Pizilla',
         'Plagger\/',
+        'Pleroma ',
         'Ploetz \+ Zeller',
         'Plukkie',
         'plumanalytics',
         'PocketImageCache',
         'PocketParser',
         'Pockey',
+        'PodcastAddict\/',
         'POE-Component-Client-HTTP',
         'Polymail\/',
         'Pompos',
         'Porkbun',
         'Port Monitor',
         'postano',
+        'postfix-mta-sts-resolver',
         'PostmanRuntime',
+        'postplanner\.com',
         'PostPost',
         'postrank',
         'PowerPoint\/',
+        'Prebid',
+        'Prerender',
         'Priceonomics Analysis Engine',
         'PrintFriendly',
         'PritTorrent',
         'Prlog',
         'probethenet',
-        'Project 25499',
+        'Project ?25499',
+        'Project-Resonance',
         'prospectb2b',
         'Protopage',
         'ProWebWalker',
@@ -826,9 +966,9 @@ class Crawlers extends AbstractProvider
         'pshtt, https scanning',
         'PTST ',
         'PTST\/[0-9]+',
-        'Pulsepoint XT3 web scraper',
         'Pump',
         'Python-httplib2',
+        'python-httpx',
         'python-requests',
         'Python-urllib',
         'Qirina Hurdler',
@@ -838,12 +978,17 @@ class Crawlers extends AbstractProvider
         'Qualidator',
         'QueryN Metasearch',
         'queuedriver',
+        'quic-go-HTTP\/',
+        'QuiteRSS',
         'Quora Link Preview',
         'Qwantify',
         'Radian6',
+        'RadioPublicImageResizer',
+        'Railgun\/',
         'RankActive',
         'RankFlex',
         'RankSonicSiteAuditor',
+        'RapidLoad\/',
         'Re-re Studio',
         'ReactorNetty',
         'Readability',
@@ -854,6 +999,7 @@ class Crawlers extends AbstractProvider
         'RecurPost\/',
         'redback\/',
         'ReederForMac',
+        'Reeder\/',
         'ReGet',
         'RepoMonkey',
         'request\.js',
@@ -866,38 +1012,44 @@ class Crawlers extends AbstractProvider
         'Robozilla',
         'ROI Hunter',
         'RPT-HTTPClient',
+        'RSSMix\/',
         'RSSOwl',
+        'RyowlEngine',
         'safe-agent-scanner',
         'SalesIntelligent',
         'Saleslift',
-        'Sendsay\.Ru',
+        'SAP NetWeaver Application Server',
         'SauceNAO',
         'SBIder',
+        'sc-downloader',
         'scalaj-http',
-        'scan\.lol',
+        'Scamadviser-Frontend',
         'ScanAlert',
+        'scan\.lol',
         'Scoop',
         'scooter',
+        'ScopeContentAG-HTTP-Client',
         'ScoutJet',
         'ScoutURLMonitor',
         'ScrapeBox Page Scanner',
-        'SimpleScraper',
         'Scrapy',
         'Screaming',
         'ScreenShotService',
         'Scrubby',
         'Scrutiny\/',
-        'search\.thunderstone',
         'Search37',
         'searchenginepromotionhelp',
         'Searchestate',
         'SearchExpress',
         'SearchSight',
+        'SearchWP',
+        'search\.thunderstone',
         'Seeker',
         'semanticdiscovery',
         'semanticjuice',
         'Semiocast HTTP client',
         'Semrush',
+        'Sendsay\.Ru',
         'sentry\/',
         'SEO Browser',
         'Seo Servis',
@@ -906,11 +1058,13 @@ class Crawlers extends AbstractProvider
         'Seobility',
         'SEOCentro',
         'SeoCheck',
+        'seocompany',
         'SEOkicks',
+        'SEOlizer',
         'Seomoz',
         'SEOprofiler',
-        'SEOsearch',
         'seoscanners',
+        'SEOsearch',
         'seositecheckup',
         'SEOstats',
         'servernfo',
@@ -921,14 +1075,16 @@ class Crawlers extends AbstractProvider
         'Shoppimon',
         'ShopWiki',
         'ShortLinkTranslate',
+        'shortURL lengthener',
         'shrinktheweb',
         'Sideqik',
+        'Siege',
         'SimplePie',
         'SimplyFast',
         'Siphon',
         'SISTRIX',
-        'Site-Shot\/',
         'Site Sucker',
+        'Site-Shot\/',
         'Site24x7',
         'SiteBar',
         'Sitebeam',
@@ -949,6 +1105,7 @@ class Crawlers extends AbstractProvider
         'sitexy\.com',
         'SkypeUriPreview',
         'Slack\/',
+        'sli-systems\.com',
         'slider\.com',
         'slurp',
         'SlySearch',
@@ -971,9 +1128,11 @@ class Crawlers extends AbstractProvider
         'SpamExperts',
         'Spammen',
         'Spanner',
+        'Spawning-AI',
         'spaziodati',
         'SPDYCheck',
         'Specificfeeds',
+        'SpeedKit',
         'speedy',
         'SPEng',
         'Spinn3r',
@@ -987,9 +1146,11 @@ class Crawlers extends AbstractProvider
         'ssl-tools',
         'StackRambler',
         'Statastico\/',
+        'Statically-',
         'StatusCake',
         'Steeler',
         'Stratagems Kumo',
+        'Stripe\/',
         'Stroke\.cz',
         'StudioFACA',
         'StumbleUpon',
@@ -999,9 +1160,11 @@ class Crawlers extends AbstractProvider
         'SuperHTTP',
         'Surphace Scout',
         'Suzuran',
-        'SwiteScraper',
+        'swcd ',
         'Symfony BrowserKit',
         'Symfony2 BrowserKit',
+        'Synapse\/',
+        'Syndirella\/',
         'SynHttpClient-Built',
         'Sysomos',
         'sysscan',
@@ -1011,6 +1174,8 @@ class Crawlers extends AbstractProvider
         'Tarantula\/',
         'Taringa UGC',
         'TarmotGezgin',
+        'tchelebi\.io',
+        'techiaith\.cymru',
         'Teleport',
         'Telesoft',
         'Telesphoreo',
@@ -1021,13 +1186,15 @@ class Crawlers extends AbstractProvider
         'Test Certificate Info',
         'testuri',
         'Tetrahedron',
+        'TextRazor Downloader',
         'The Drop Reaper',
         'The Expert HTML Source Viewer',
-        'The Knowledge AI',
         'The Intraformant',
+        'The Knowledge AI',
         'theinternetrules',
         'TheNomad',
         'Thinklab',
+        'Thumbor',
         'Thumbshots',
         'ThumbSniper',
         'timewe\.net',
@@ -1061,9 +1228,11 @@ class Crawlers extends AbstractProvider
         'ubermetrics-technologies',
         'uclassify',
         'UdmSearch',
+        'ultimate_sitemap_parser',
         'unchaos',
         'unirest-java',
         'UniversalFeedParser',
+        'unshortenit',
         'Unshorten\.It',
         'Untiny',
         'UnwindFetchor',
@@ -1071,8 +1240,8 @@ class Crawlers extends AbstractProvider
         'updown\.io daemon',
         'Upflow',
         'Uptimia',
-        'Urlcheckr',
         'URL Verifier',
+        'Urlcheckr',
         'URLitor',
         'urlresolver',
         'Urlstat',
@@ -1080,15 +1249,16 @@ class Crawlers extends AbstractProvider
         'UrlTrends Ranking Updater',
         'URLy Warning',
         'URLy\.Warning',
+        'URL\/Emacs',
         'Vacuum',
         'Vagabondo',
         'VB Project',
         'vBSEO',
         'VCI',
         'via ggpht\.com GoogleImageProxy',
-        'VidibleScraper',
         'Virusdie',
         'visionutils',
+        'Visual Rights Group',
         'vkShare',
         'VoidEYE',
         'Voil',
@@ -1099,19 +1269,18 @@ class Crawlers extends AbstractProvider
         'Vulnbusters Meter',
         'VYU2',
         'w3af\.org',
-        'W3C_Unicorn',
         'W3C-checklink',
         'W3C-mobileOK',
+        'W3C_Unicorn',
         'WAC-OFU',
-        'Wallpapers\/[0-9]+',
+        'WakeletLinkExpander',
         'WallpapersHD',
+        'Wallpapers\/[0-9]+',
         'wangling',
         'Wappalyzer',
         'WatchMouse',
         'WbSrch\/',
         'WDT\.io',
-        'web-capture\.net',
-        'Web-sniffer',
         'Web Auto',
         'Web Collage',
         'Web Enhancer',
@@ -1119,7 +1288,10 @@ class Crawlers extends AbstractProvider
         'Web Fuck',
         'Web Pix',
         'Web Sauger',
+        'Web spyder',
         'Web Sucker',
+        'web-capture\.net',
+        'Web-sniffer',
         'Webalta',
         'Webauskunft',
         'WebAuto',
@@ -1157,8 +1329,9 @@ class Crawlers extends AbstractProvider
         'Webster',
         'WebStripper',
         'WebSucker',
-        'Webthumb\/',
+        'webtech\/',
         'WebThumbnail',
+        'Webthumb\/',
         'WebWhacker',
         'WebZIP',
         'WeLikeLinks',
@@ -1167,24 +1340,31 @@ class Crawlers extends AbstractProvider
         'wf84',
         'Wfuzz\/',
         'wget',
+        'WhatCMS',
         'WhatsApp',
         'WhatsMyIP',
         'WhatWeb',
         'WhereGoes\?',
         'Whibse',
+        'WhoAPI\/',
         'WhoRunsCoinHive',
         'Whynder Magnet',
         'Windows-RSS-Platform',
+        'WinHttp-Autoproxy-Service',
+        'WinHTTP\/',
         'WinPodder',
         'wkhtmlto',
         'wmtips',
         'Woko',
+        'Wolfram HTTPClient',
         'woorankreview',
-        'Word\/',
         'WordPress\/',
         'WordupinfoSearch',
+        'Word\/',
+        'worldping-api',
         'wotbox',
         'WP Engine Install Performance API',
+        'WP Rocket',
         'wpif',
         'wprecon\.com survey',
         'WPScan',
@@ -1194,20 +1374,21 @@ class Crawlers extends AbstractProvider
         'WWW-Mechanize',
         'WWW::Document',
         'WWW::Mechanize',
-        'www\.monitor\.us',
         'WWWOFFLE',
+        'www\.monitor\.us',
         'x09Mozilla',
         'x22Mozilla',
         'XaxisSemanticsClassifier',
+        'XenForo\/',
         'Xenu Link Sleuth',
         'XING-contenttabreceiver',
         'xpymep([0-9]?)\.exe',
-        'Y!J-(ASR|BSC)',
-        'Y\!J-BRW',
+        'Y!J-[A-Z][A-Z][A-Z]',
         'Yaanb',
         'yacy',
         'Yahoo Link Preview',
         'YahooCacheSystem',
+        'YahooMailProxy',
         'YahooYSMcm',
         'YandeG',
         'Yandex(?!Search)',
@@ -1215,6 +1396,7 @@ class Crawlers extends AbstractProvider
         'yeti',
         'Yo-yo',
         'Yoleo Consumer',
+        'yomins\.com',
         'yoogliFetchAgent',
         'YottaaMonitor',
         'Your-Website-Sucks',
@@ -1226,15 +1408,17 @@ class Crawlers extends AbstractProvider
         'Zao',
         'Zauba',
         'Zemanta Aggregator',
-        'Zend_Http_Client',
         'Zend\\\\Http\\\\Client',
+        'Zend_Http_Client',
         'Zermelo',
         'Zeus ',
         'zgrab',
         'ZnajdzFoto',
+        'ZnHTTP',
         'Zombie\.js',
         'Zoom\.Mac',
+        'ZoteroTranslationServer',
         'ZyBorg',
-        '[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer)',
+        '[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)',
     );
 }
index e6b3ca8978aba097e4edb51fa2e62616f5ca38f3..62745572c8d22b835f9f45908bb3b3e4168e280d 100644 (file)
@@ -67,6 +67,8 @@ class Exclusions extends AbstractProvider
         '; IDbot',
         '; ID bot',
         '; POWER BOT',
-        ';', // Remove the following characters ;
+        'OCTOPUS-CORE',
+        'htc_botdugls',
+        'super\/\d+\/Android\/\d+',
     );
 }