]> git.mxchange.org Git - hub.git/commitdiff
Continued:
authorRoland Häder <roland@mxchange.org>
Mon, 4 Jan 2021 02:29:32 +0000 (03:29 +0100)
committerRoland Häder <roland@mxchange.org>
Wed, 6 Jan 2021 19:32:43 +0000 (20:32 +0100)
- renamed default crawler names to 'distributed' as this is now the crawler's
  default mode. Later more modes, e.g. 'local' can be added which won't register
  any remote jobs and maybe not communicate with the node (hub) to avoid crawler
  info distribution
- Just a wild idea!
- updated core framework

Signed-off-by: Roland Häder <roland@mxchange.org>
application/hub/classes/commands/console/class_HubConsoleCrawlerCommand.php
application/hub/classes/crawler/console/class_NodeConsoleCrawler.php [deleted file]
application/hub/classes/crawler/console/class_NodeConsoleDistributedCrawler.php [new file with mode: 0644]
application/hub/classes/filter/class_BaseHubFilter.php
application/hub/classes/filter/crawler/class_CrawlerInitializationFilter.php
application/hub/classes/filter/task/class_BaseCrawlerTaskFilter.php [new file with mode: 0644]
application/hub/classes/filter/task/crawler/class_CrawlerDistributedTaskHandlerInitializerFilter.php [new file with mode: 0644]
application/hub/classes/filter/task/crawler/class_CrawlerTaskHandlerInitializerFilter.php [deleted file]
application/hub/classes/source/urls/class_CrawlerUploadedListUrlSource.php
application/hub/config.php
core

index 6b02bc3b59e74c3ad720d92545147c4288860d72..eb9df8db3289dc8dd5a83b49f292795c55390a9f 100644 (file)
@@ -127,7 +127,6 @@ class HubConsoleCrawlerCommand extends BaseCommand implements Commandable {
                $controllerInstance->addPreFilter(ObjectFactory::createObjectByConfiguredName('crawler_welcome_teaser_filter_class'));
 
                // Add bootstrap filters
-               $controllerInstance->addBootstrapFilter(ObjectFactory::createObjectByConfiguredName('crawler_bootstrap_task_handler_initializer_filter_class'));
                $controllerInstance->addBootstrapFilter(ObjectFactory::createObjectByConfiguredName('crawler_bootstrap_extra_bootstrapping_filter_class'));
 
                // This is the last generic boostrap filter
diff --git a/application/hub/classes/crawler/console/class_NodeConsoleCrawler.php b/application/hub/classes/crawler/console/class_NodeConsoleCrawler.php
deleted file mode 100644 (file)
index d834b36..0000000
+++ /dev/null
@@ -1,101 +0,0 @@
-<?php
-// Own namespace
-namespace Org\Shipsimu\Hub\Crawler;
-
-// Import framework stuff
-use Org\Mxchange\CoreFramework\Controller\Controller;
-use Org\Mxchange\CoreFramework\Helper\Application\ApplicationHelper;
-use Org\Mxchange\CoreFramework\Registry\GenericRegistry;
-use Org\Mxchange\CoreFramework\Registry\Registerable;
-use Org\Mxchange\CoreFramework\Response\Responseable;
-
-/**
- * A node console crawler
- *
- * @author             Roland Haeder <webmaster@shipsimu.org>
- * @version            0.0.0
- * @copyright  Copyright (c) 2014 - 2020 Crawler Developer Team
- * @license            GNU GPL 3.0 or any newer version
- * @link               http://www.shipsimu.org
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-class NodeConsoleCrawler extends BaseNodeCrawler implements Crawler, Registerable {
-       /**
-        * Protected constructor
-        *
-        * @return      void
-        */
-       private function __construct () {
-               // Call parent constructor
-               parent::__construct(__CLASS__);
-       }
-
-       /**
-        * Creates an instance of this class
-        *
-        * @return      $crawlerInstance        An instance of a Crawler class
-        */
-       public final static function createNodeConsoleCrawler () {
-               // Get new instance
-               $crawlerInstance = new NodeConsoleCrawler();
-
-               // Return the prepared instance
-               return $crawlerInstance;
-       }
-
-       /**
-        * Method to "bootstrap" the crawler. This step does also apply provided
-        * command-line arguments stored in the request instance. You should now
-        * get it from calling FrameworkBootstrap::getRequestInstance().
-        *
-        * @return      void
-        */
-       public function doBootstrapping () {
-               $this->partialStub('Please implement this method.');
-       }
-
-       /**
-        * Outputs the console teaser. This should only be executed on startup or
-        * full restarts. This method generates some space around the teaser.
-        *
-        * @return      void
-        */
-       public function outputConsoleTeaser () {
-               // Get the app instance (for shortening our code)
-               $applicationInstance = ApplicationHelper::getSelfInstance();
-
-               // Output all lines
-               self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(' ');
-               self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('Distributed Crawler v' . $applicationInstance->getAppVersion());
-               self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('Copyright (c) 2014 - 2020 Crawler Developer Team');
-               self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(' ');
-               self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('This program comes with ABSOLUTELY NO WARRANTY; for details see docs/COPYING.');
-               self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('This is free software, and you are welcome to redistribute it under certain');
-               self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('conditions; see docs/COPYING for details.');
-               self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(' ');
-       }
-
-       /**
-        * Add some crawler-specific filters
-        *
-        * @param       $controllerInstance             An object of a Controller instance
-        * @param       $responseInstance               An object of a Responseable instance
-        * @return      void
-        */
-       public function addExtraFilters (Controller $controllerInstance, Responseable $responseInstance) {
-               $this->partialStub('Please implement this method.');
-       }
-
-}
diff --git a/application/hub/classes/crawler/console/class_NodeConsoleDistributedCrawler.php b/application/hub/classes/crawler/console/class_NodeConsoleDistributedCrawler.php
new file mode 100644 (file)
index 0000000..d23b257
--- /dev/null
@@ -0,0 +1,103 @@
+<?php
+// Own namespace
+namespace Org\Shipsimu\Hub\Crawler;
+
+// Import framework stuff
+use Org\Mxchange\CoreFramework\Controller\Controller;
+use Org\Mxchange\CoreFramework\Factory\Object\ObjectFactory;
+use Org\Mxchange\CoreFramework\Helper\Application\ApplicationHelper;
+use Org\Mxchange\CoreFramework\Registry\GenericRegistry;
+use Org\Mxchange\CoreFramework\Registry\Registerable;
+use Org\Mxchange\CoreFramework\Response\Responseable;
+
+/**
+ * A distributed node console crawler
+ *
+ * @author             Roland Haeder <webmaster@shipsimu.org>
+ * @version            0.0.0
+ * @copyright  Copyright (c) 2014 - 2020 Crawler Developer Team
+ * @license            GNU GPL 3.0 or any newer version
+ * @link               http://www.shipsimu.org
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+class NodeConsoleDistributedCrawler extends BaseNodeCrawler implements Crawler, Registerable {
+       /**
+        * Protected constructor
+        *
+        * @return      void
+        */
+       private function __construct () {
+               // Call parent constructor
+               parent::__construct(__CLASS__);
+       }
+
+       /**
+        * Creates an instance of this class
+        *
+        * @return      $crawlerInstance        An instance of a Crawler class
+        */
+       public final static function createNodeConsoleDistributedCrawler () {
+               // Get new instance
+               $crawlerInstance = new NodeConsoleDistributedCrawler();
+
+               // Return the prepared instance
+               return $crawlerInstance;
+       }
+
+       /**
+        * Method to "bootstrap" the crawler. This step does also apply provided
+        * command-line arguments stored in the request instance. You should now
+        * get it from calling FrameworkBootstrap::getRequestInstance().
+        *
+        * @return      void
+        */
+       public function doBootstrapping () {
+               $this->partialStub('Please implement this method.');
+       }
+
+       /**
+        * Outputs the console teaser. This should only be executed on startup or
+        * full restarts. This method generates some space around the teaser.
+        *
+        * @return      void
+        */
+       public function outputConsoleTeaser () {
+               // Get the app instance (for shortening our code)
+               $applicationInstance = ApplicationHelper::getSelfInstance();
+
+               // Output all lines
+               self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(' ');
+               self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('Distributed Crawler v' . $applicationInstance->getAppVersion());
+               self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('Copyright (c) 2014 - 2020 Crawler Developer Team');
+               self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(' ');
+               self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('This program comes with ABSOLUTELY NO WARRANTY; for details see docs/COPYING.');
+               self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('This is free software, and you are welcome to redistribute it under certain');
+               self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('conditions; see docs/COPYING for details.');
+               self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(' ');
+       }
+
+       /**
+        * Add some crawler-specific filters
+        *
+        * @param       $controllerInstance             An object of a Controller instance
+        * @param       $responseInstance               An object of a Responseable instance
+        * @return      void
+        */
+       public function addExtraFilters (Controller $controllerInstance, Responseable $responseInstance) {
+               // Add distributed crawler related bootstrap-filters
+               $controllerInstance->addBootstrapFilter(ObjectFactory::createObjectByConfiguredName('crawler_bootstrap_distributed_task_handler_initializer_filter_class'));
+       }
+
+}
index fbf5e822a0f6b7b3d1c04738d0d69049da05046a..d31b697bef60cb395dbf3627c6e6f0cce6bc6338 100644 (file)
@@ -110,7 +110,7 @@ abstract class BaseHubFilter extends BaseFilter {
                }
 
                // Construct an array for pushing it on next stack
-               $messageArray = array(
+               $messageArray = [
                        // Message data itself
                        NetworkPackageHandler::MESSAGE_ARRAY_DATA   => $this->dataXmlNodes,
                        // Message type (which is $messageType)
@@ -121,7 +121,7 @@ abstract class BaseHubFilter extends BaseFilter {
                        NetworkPackageHandler::MESSAGE_ARRAY_HASH   => $messageData[NetworkPackageHandler::PACKAGE_CONTENT_HASH],
                        // Package tags
                        NetworkPackageHandler::MESSAGE_ARRAY_TAGS   => $messageData[NetworkPackageHandler::PACKAGE_CONTENT_TAGS],
-               );
+               ];
 
                // Push the processed message back on stack
                $handlerInstance->getStackInstance()->pushNamed(NetworkPackageHandler::STACKER_NAME_PROCESSED_MESSAGE, $messageArray);
index 3f1f6ac5b7751126f748002be869edb460c10a54..fee0f05fedc2234d5e088228394c5a64e4d93ff1 100644 (file)
@@ -14,7 +14,7 @@ use Org\Mxchange\CoreFramework\Request\Requestable;
 use Org\Mxchange\CoreFramework\Response\Responseable;
 
 /**
- * A Initialization filter for crawler
+ * An iitialization filter for crawlers
  *
  * @author             Roland Haeder <webmaster@shipsimu.org>
  * @version            0.0.0
@@ -68,8 +68,17 @@ class CrawlerInitializationFilter extends BaseCrawlerFilter implements Filterabl
         * @todo        0% done
         */
        public function execute (Requestable $requestInstance, Responseable $responseInstance) {
+               // Default crawler mode is full-blown flag ship!
+               $crawlerMode = 'distributed';
+
+               // Is 'mode' set?
+               if ($requestInstance->isRequestElementSet('mode')) {
+                       // Get 'mode' parameter
+                       $crawlerMode = $requestInstance->getRequestElement('mode');
+               }
+
                // Get an instance
-               $crawlerInstance = ObjectFactory::createObjectByConfiguredName('node_console_crawler_class');
+               $crawlerInstance = ObjectFactory::createObjectByConfiguredName(sprintf('node_console_%s_crawler_class', $crawlerMode));
 
                // Get a registry
                $applicationInstance = ApplicationHelper::getSelfInstance();
diff --git a/application/hub/classes/filter/task/class_BaseCrawlerTaskFilter.php b/application/hub/classes/filter/task/class_BaseCrawlerTaskFilter.php
new file mode 100644 (file)
index 0000000..cbb53b6
--- /dev/null
@@ -0,0 +1,85 @@
+<?php
+// Own namespace
+namespace Org\Shipsimu\Hub\Crawler\Filter\Task;
+
+// Import application-specific stuff
+use Org\Shipsimu\Hub\Crawler\Filter\BaseCrawlerFilter;
+
+// Import framework stuff
+use Org\Mxchange\CoreFramework\Factory\Object\ObjectFactory;
+use Org\Mxchange\CoreFramework\Registry\GenericRegistry;
+use Org\Mxchange\CoreFramework\Request\Requestable;
+use Org\Mxchange\CoreFramework\Response\Responseable;
+
+/**
+ * A generic filter for crawler tasks
+ *
+ * @author             Roland Haeder <webmaster@shipsimu.org>
+ * @version            0.0.0
+ * @copyright  Copyright (c) 2014 - 2020 Crawler Developer Team
+ * @license            GNU GPL 3.0 or any newer version
+ * @link               http://www.shipsimu.org
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+abstract class BaseCrawlerTaskFilter extends BaseCrawlerFilter {
+       /**
+        * Protected constructor
+        *
+        * @param       $className      Real name of class
+        * @return      void
+        */
+       protected function __construct (string $className) {
+               // Call parent constructor
+               parent::__construct($className);
+       }
+
+       /**
+        * Executes the filter with given request and response objects
+        *
+        * @param       $requestInstance        An instance of a class with an Requestable interface
+        * @param       $responseInstance       An instance of a class with an Responseable interface
+        * @return      void
+        * @throws      FilterChainException    If the filter chain needs to be interrupted
+        * @todo        10% done
+        */
+       public function execute (Requestable $requestInstance, Responseable $responseInstance) {
+               // Get a new task handler instance
+               $handlerInstance = ObjectFactory::createObjectByConfiguredName('task_handler_class');
+
+               /*
+                * Register tasks for all crawlers:
+                *
+                * 1) MIME sniffer
+                */
+               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_mime_sniffer_task_class');
+               $handlerInstance->registerTask('crawler_mime_sniffer', $taskInstance);
+
+               // 2) Document parser (converts document to meta format)
+               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_document_parser_task_class');
+               $handlerInstance->registerTask('crawler_document_parser', $taskInstance);
+
+               // 3) Document structure analyzer
+               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_structure_analyzer_task_class');
+               $handlerInstance->registerTask('crawler_structure_analyzer', $taskInstance);
+
+               // 4) Snippet extractor
+               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_snippet_extractor_task_class');
+               $handlerInstance->registerTask('crawler_snippet_extractor', $taskInstance);
+
+               // Add handler instance to registry
+               GenericRegistry::getRegistry()->addInstance('task_handler', $handlerInstance);
+       }
+
+}
diff --git a/application/hub/classes/filter/task/crawler/class_CrawlerDistributedTaskHandlerInitializerFilter.php b/application/hub/classes/filter/task/crawler/class_CrawlerDistributedTaskHandlerInitializerFilter.php
new file mode 100644 (file)
index 0000000..a9d1d99
--- /dev/null
@@ -0,0 +1,112 @@
+<?php
+// Own namespace
+namespace Org\Shipsimu\Hub\Filter\Initializer\Crawler\Handler\Task;
+
+// Import application-specific stuff
+use Org\Shipsimu\Hub\Crawler\Filter\Task\BaseCrawlerTaskFilter;
+
+// Import framework stuff
+use Org\Mxchange\CoreFramework\Bootstrap\FrameworkBootstrap;
+use Org\Mxchange\CoreFramework\Factory\Object\ObjectFactory;
+use Org\Mxchange\CoreFramework\Filter\Filterable;
+use Org\Mxchange\CoreFramework\Registry\GenericRegistry;
+use Org\Mxchange\CoreFramework\Request\Requestable;
+use Org\Mxchange\CoreFramework\Response\Responseable;
+
+/**
+ * A TaskHandlerInitializer filter for crawler
+ *
+ * @author             Roland Haeder <webmaster@shipsimu.org>
+ * @version            0.0.0
+ * @copyright  Copyright (c) 2014 - 2020 Crawler Developer Team
+ * @license            GNU GPL 3.0 or any newer version
+ * @link               http://www.shipsimu.org
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+class CrawlerDistributedTaskHandlerInitializerFilter extends BaseCrawlerTaskFilter implements Filterable {
+       /**
+        * Protected constructor
+        *
+        * @return      void
+        */
+       private function __construct () {
+               // Call parent constructor
+               parent::__construct(__CLASS__);
+       }
+
+       /**
+        * Creates an instance of this filter class
+        *
+        * @return      $filterInstance         An instance of this filter class
+        */
+       public static final function createCrawlerDistributedTaskHandlerInitializerFilter () {
+               // Get a new instance
+               $filterInstance = new CrawlerDistributedTaskHandlerInitializerFilter();
+
+               // Return the instance
+               return $filterInstance;
+       }
+
+       /**
+        * Executes the filter with given request and response objects
+        *
+        * @param       $requestInstance        An instance of a class with an Requestable interface
+        * @param       $responseInstance       An instance of a class with an Responseable interface
+        * @return      void
+        * @throws      FilterChainException    If the filter chain needs to be interrupted
+        * @todo        10% done
+        */
+       public function execute (Requestable $requestInstance, Responseable $responseInstance) {
+               // Run parent method first
+               parent::execute($requestInstance, $responseInstance);
+
+               // Get task handler instance
+               $handlerInstance = GenericRegistry::getRegistry()->getInstance('task_handler');
+
+               /*
+                * Register all tasks for a distributed crawler:
+                *
+                * 1) "Communicator" for crawler->node communication
+                */
+               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_node_communicator_task_class');
+               $handlerInstance->registerTask('crawler_node_communicator', $taskInstance);
+
+               // 2) Local URL crawler (gets URLs locally and adds them to the analyzer's input stack)
+               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_local_url_crawler_task_class');
+               $handlerInstance->registerTask('crawler_local_url_crawler', $taskInstance);
+
+               // 3) Remote URL crawler (gets URLs locally for other nodes, also includes the crawled URL in local index)
+               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_remote_url_crawler_task_class');
+               $handlerInstance->registerTask('crawler_remote_url_crawler', $taskInstance);
+
+               // 4) Remote-crawl publisher (publishes crawl jobs for remote retrieval)
+               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_remote_job_publisher_task_class');
+               $handlerInstance->registerTask('crawler_remote_job_publisher', $taskInstance);
+
+               // 5) Node ping
+               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_ping_task_class');
+               $handlerInstance->registerTask('crawler_ping', $taskInstance);
+
+               // 6) URL sources
+               foreach (explode(':', FrameworkBootstrap::getConfigurationInstance()->getConfigEntry('crawler_url_stacks')) as $stack) {
+                       // Init task instance
+                       $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_url_source_' . $stack . '_task_class');
+
+                       // And register it
+                       $handlerInstance->registerTask('crawler_url_source_' . $stack, $taskInstance);
+               }
+       }
+
+}
diff --git a/application/hub/classes/filter/task/crawler/class_CrawlerTaskHandlerInitializerFilter.php b/application/hub/classes/filter/task/crawler/class_CrawlerTaskHandlerInitializerFilter.php
deleted file mode 100644 (file)
index 8a264df..0000000
+++ /dev/null
@@ -1,131 +0,0 @@
-<?php
-// Own namespace
-namespace Org\Shipsimu\Hub\Filter\Initializer\Crawler\Handler\Task;
-
-// Import application-specific stuff
-use Org\Shipsimu\Hub\Crawler\Filter\BaseCrawlerFilter;
-
-// Import framework stuff
-use Org\Mxchange\CoreFramework\Bootstrap\FrameworkBootstrap;
-use Org\Mxchange\CoreFramework\Factory\Object\ObjectFactory;
-use Org\Mxchange\CoreFramework\Filter\Filterable;
-use Org\Mxchange\CoreFramework\Registry\GenericRegistry;
-use Org\Mxchange\CoreFramework\Request\Requestable;
-use Org\Mxchange\CoreFramework\Response\Responseable;
-
-/**
- * A TaskHandlerInitializer filter for crawler
- *
- * @author             Roland Haeder <webmaster@shipsimu.org>
- * @version            0.0.0
- * @copyright  Copyright (c) 2014 - 2020 Crawler Developer Team
- * @license            GNU GPL 3.0 or any newer version
- * @link               http://www.shipsimu.org
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-class CrawlerTaskHandlerInitializerFilter extends BaseCrawlerFilter implements Filterable {
-       /**
-        * Protected constructor
-        *
-        * @return      void
-        */
-       private function __construct () {
-               // Call parent constructor
-               parent::__construct(__CLASS__);
-       }
-
-       /**
-        * Creates an instance of this filter class
-        *
-        * @return      $filterInstance         An instance of this filter class
-        */
-       public static final function createCrawlerTaskHandlerInitializerFilter () {
-               // Get a new instance
-               $filterInstance = new CrawlerTaskHandlerInitializerFilter();
-
-               // Return the instance
-               return $filterInstance;
-       }
-
-       /**
-        * Executes the filter with given request and response objects
-        *
-        * @param       $requestInstance        An instance of a class with an Requestable interface
-        * @param       $responseInstance       An instance of a class with an Responseable interface
-        * @return      void
-        * @throws      FilterChainException    If the filter chain needs to be interrupted
-        * @todo        10% done
-        */
-       public function execute (Requestable $requestInstance, Responseable $responseInstance) {
-               // Get crawler instance
-               $crawlerInstance = GenericRegistry::getRegistry()->getInstance('crawler');
-
-               // Get a new task handler instance
-               $handlerInstance = ObjectFactory::createObjectByConfiguredName('task_handler_class');
-
-               /*
-                * Register all tasks:
-                *
-                * 1) "Communicator" for crawler->node communication
-                */
-               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_node_communicator_task_class');
-               $handlerInstance->registerTask('crawler_node_communicator', $taskInstance);
-
-               // 2) Local URL crawler (gets URLs locally and adds them to the analyzer's input stack)
-               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_local_url_crawler_task_class');
-               $handlerInstance->registerTask('crawler_local_url_crawler', $taskInstance);
-
-               // 3) Remote URL crawler (gets URLs locally for other nodes, also includes the crawled URL in local index)
-               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_remote_url_crawler_task_class');
-               $handlerInstance->registerTask('crawler_remote_url_crawler', $taskInstance);
-
-               // 4) Remote-crawl publisher (publishes crawl jobs for remote retrieval)
-               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_remote_job_publisher_task_class');
-               $handlerInstance->registerTask('crawler_remote_job_publisher', $taskInstance);
-
-               // 5) MIME sniffer
-               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_mime_sniffer_task_class');
-               $handlerInstance->registerTask('crawler_mime_sniffer', $taskInstance);
-
-               // 6) Document parser (converts document to meta format)
-               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_document_parser_task_class');
-               $handlerInstance->registerTask('crawler_document_parser', $taskInstance);
-
-               // 7) Document structure analyzer
-               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_structure_analyzer_task_class');
-               $handlerInstance->registerTask('crawler_structure_analyzer', $taskInstance);
-
-               // 8) Snippet extractor
-               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_snippet_extractor_task_class');
-               $handlerInstance->registerTask('crawler_snippet_extractor', $taskInstance);
-
-               // 9) Node ping
-               $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_ping_task_class');
-               $handlerInstance->registerTask('crawler_ping', $taskInstance);
-
-               // 10) URL sources
-               foreach (explode(':', FrameworkBootstrap::getConfigurationInstance()->getConfigEntry('crawler_url_stacks')) as $stack) {
-                       // Init task instance
-                       $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_url_source_' . $stack . '_task_class');
-
-                       // And register it
-                       $handlerInstance->registerTask('crawler_url_source_' . $stack, $taskInstance);
-               }
-
-               // Put the task handler in registry
-               GenericRegistry::getRegistry()->addInstance('task_handler', $handlerInstance);
-       }
-
-}
index 9830b7d8ba5bc0223ecb5e47e0e49cbbb3dbf863..f012e66fc8f1197d709d2c422d6de9df5343d65e 100644 (file)
@@ -164,7 +164,7 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R
 
                // Read next entry
                /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('CRAWLER-UPLOADED-LIST-URL-SOURCE: this->csvFileImported=%s', print_r($this->csvFileImported, TRUE)));
-               $directoryEntry = $this->getDirectoryInstance()->readDirectoryExcept(array_merge(['.htaccess', '.', '..'], $this->csvFileImported));
+               $directoryEntry = $this->getDirectoryInstance()->readDirectoryExcept(array_merge(['.gitkeep', '.', '..'], $this->csvFileImported));
 
                // Is it empty or wrong file extension?
                /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('CRAWLER-UPLOADED-LIST-URL-SOURCE: directoryEntry(%d)=%s', strlen($directoryEntry), $directoryEntry));
@@ -206,7 +206,7 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R
                $csvArray = [
                        UrlSource::URL_SOURCE_ARRAY_START_URL      => $csvData[0],
                        UrlSource::URL_SOURCE_ARRAY_START_DEPTH    => $csvData[1],
-                       UrlSource::URL_SOURCE_ARRAY_EXTERNAL_DEPTH => $csvData[2]
+                       UrlSource::URL_SOURCE_ARRAY_EXTERNAL_DEPTH => $csvData[2],
                ];
 
                // Then add more data to it
index 45cf27b1659a0b631bac530f50b4299a50c0d889..80511a03c5cb5366441773e4cc2f358b869186b5 100644 (file)
@@ -171,73 +171,73 @@ $cfg->setConfigEntry('hub_html_controller_resolver_class', 'Org\Shipsimu\Hub\Res
 // CFG: HUB-CONSOLE-CMD-MAIN-RESOLVER-CLASS
 $cfg->setConfigEntry('hub_console_cmd_main_resolver_class', 'Org\Shipsimu\Hub\Resolver\Command\HubConsoleCommandResolver');
 
-// CFG: NEWS-DOWNLOAD-FILTER
+// CFG: NEWS-DOWNLOAD-FILTER-CLASS
 $cfg->setConfigEntry('news_download_filter_class', 'Org\Mxchange\CoreFramework\Filter\News\NewsDownloadFilter');
 
-// CFG: NEWS-PROCESS-FILTER
+// CFG: NEWS-PROCESS-FILTER-CLASS
 $cfg->setConfigEntry('news_process_filter_class', 'Org\Mxchange\CoreFramework\Filter\News\NewsProcessFilter');
 
-// CFG: NODE-PHP-REQUIREMENTS-FILTER
+// CFG: NODE-PHP-REQUIREMENTS-FILTER-CLASS
 $cfg->setConfigEntry('node_php_requirements_filter_class', 'Org\Shipsimu\Hub\Filter\Node\Requirements\NodePhpRequirementsFilter');
 
-// CFG: NODE-INITIALIZER-FILTER
+// CFG: NODE-INITIALIZER-FILTER-CLASS
 $cfg->setConfigEntry('node_initializer_filter_class', 'Org\Shipsimu\Hub\Filter\Node\Initialization\NodeInitializationFilter');
 
-// CFG: NODE-ACTIVATION-TASK-HANDLER-INITIALIZER-FILTER
+// CFG: NODE-ACTIVATION-TASK-HANDLER-INITIALIZER-FILTER-CLASS
 $cfg->setConfigEntry('node_activation_task_handler_initializer_filter_class', 'Org\Shipsimu\Hub\Filter\Initializer\Node\Handler\Task\NodeTaskHandlerInitializerFilter');
 
-// CFG: NODE-ACTIVATION-ANNOUNCEMENT-TASK-FILTER
+// CFG: NODE-ACTIVATION-ANNOUNCEMENT-TASK-FILTER-CLASS
 $cfg->setConfigEntry('node_activation_announcement_task_filter_class', 'Org\Shipsimu\Hub\Filter\Activation\Node\Announcement\NodeActivationAnnouncementFilter');
 
-// CFG: NODE-WELCOME-TEASER-FILTER
+// CFG: NODE-WELCOME-TEASER-FILTER-CLASS
 $cfg->setConfigEntry('node_welcome_teaser_filter_class', 'Org\Shipsimu\Hub\Filter\Node\WelcomeTeaser\NodeWelcomeTeaserFilter');
 
-// CFG: NODE-BOOTSTRAP-GENERATE-NODEID-FILTER
+// CFG: NODE-BOOTSTRAP-GENERATE-NODEID-FILTER-CLASS
 $cfg->setConfigEntry('node_bootstrap_generate_nodeid_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Node\NodeId\NodeBootstrapGenerateNodeIdFilter');
 
-// CFG: NODE-BOOTSTRAP-GENERATE-SESSIONID-FILTER
+// CFG: NODE-BOOTSTRAP-GENERATE-SESSIONID-FILTER-CLASS
 $cfg->setConfigEntry('node_bootstrap_generate_sessionid_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Node\SessionId\NodeBootstrapGenerateSessionIdFilter');
 
-// CFG: NODE-BOOTSTRAP-GENERATE-PRIVATE-KEY-FILTER
+// CFG: NODE-BOOTSTRAP-GENERATE-PRIVATE-KEY-FILTER-CLASS
 $cfg->setConfigEntry('node_bootstrap_generate_private_key_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Node\PrivateKey\NodeBootstrapGeneratePrivateKeyFilter');
 
-// CFG: NODE-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER
+// CFG: NODE-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER-CLASS
 $cfg->setConfigEntry('node_bootstrap_extra_bootstrapping_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Node\Extra\NodeBootstrapExtraBootstrappingFilter');
 
-// CFG: NODE-BOOTSTRAP-LISTENER-POOL-FILTER
+// CFG: NODE-BOOTSTRAP-LISTENER-POOL-FILTER-CLASS
 $cfg->setConfigEntry('node_bootstrap_listener_pool_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Node\Pool\NodeBootstrapListenerPoolFilter');
 
-// CFG: NODE-SHUTDOWN-FLUSH-NODE-LIST-FILTER
+// CFG: NODE-SHUTDOWN-FLUSH-NODE-LIST-FILTER-CLASS
 $cfg->setConfigEntry('node_shutdown_flush_node_list_filter_class', 'Org\Shipsimu\Hub\Filter\Shutdown\Node\NodeList\NodeShutdownFlushNodeListFilter');
 
-// CFG: NODE-SHUTDOWN-TASK-HANDLER-FILTER
+// CFG: NODE-SHUTDOWN-TASK-HANDLER-FILTER-CLASS
 $cfg->setConfigEntry('node_shutdown_task_handler_filter_class', 'Org\Shipsimu\Hub\Filter\Shutdown\Node\Handler\Task\NodeShutdownTaskHandlerFilter');
 
-// CFG: NODE-SHUTDOWN-NODE-FILTER
+// CFG: NODE-SHUTDOWN-NODE-FILTER-CLASS
 $cfg->setConfigEntry('node_shutdown_node_filter_class', 'Org\Shipsimu\Hub\Filter\Shutdown\Node\NodeShutdownNodeFilter');
 
-// CFG: PACKAGE-FILTER-CHAIN-CLASS
+// CFG: PACKAGE-FILTER-CLASS-CHAIN-CLASS
 $cfg->setConfigEntry('package_filter_chain_class', 'Org\Shipsimu\Hub\FilterChain\Package\PackageFilterChain');
 
-// CFG: PACKAGE-TAG-ANNOUNCEMENT-FILTER
+// CFG: PACKAGE-TAG-ANNOUNCEMENT-FILTER-CLASS
 $cfg->setConfigEntry('package_tag_announcement_filter_class', 'Org\Shipsimu\Hub\Filter\Tag\Network\Announcement\PackageAnnouncementTagFilter');
 
-// CFG: PACKAGE-TAG-ANNOUNCEMENT-ANSWER-FILTER
+// CFG: PACKAGE-TAG-ANNOUNCEMENT-ANSWER-FILTER-CLASS
 $cfg->setConfigEntry('package_tag_announcement_answer_filter_class', 'Org\Shipsimu\Hub\Filter\Tag\Network\Answer\Announcement\PackageAnnouncementAnswerTagFilter');
 
-// CFG: PACKAGE-TAG-DHT-BOOTSTRAP-ANSWER-FILTER
+// CFG: PACKAGE-TAG-DHT-BOOTSTRAP-ANSWER-FILTER-CLASS
 $cfg->setConfigEntry('package_tag_dht_bootstrap_answer_filter_class', 'Org\Shipsimu\Hub\Filter\Tag\Network\Answer\Dht\Bootstrap\PackageDhtBootstrapAnswerTagFilter');
 
-// CFG: PACKAGE-TAG-SELF-CONNECT-FILTER
+// CFG: PACKAGE-TAG-SELF-CONNECT-FILTER-CLASS
 $cfg->setConfigEntry('package_tag_self_connect_filter_class', 'Org\Shipsimu\Hub\Filter\Tag\Network\SelfConnect\PackageSelfConnectTagFilter');
 
-// CFG: PACKAGE-TAG-REQUEST-NODE-LIST-FILTER
+// CFG: PACKAGE-TAG-REQUEST-NODE-LIST-FILTER-CLASS
 $cfg->setConfigEntry('package_tag_request_node_list_filter_class', 'Org\Shipsimu\Hub\Filter\Tag\Network\Request\NodeList\PackageRequestNodeListTagFilter');
 
-// CFG: PACKAGE-TAG-REQUEST-NODE-LIST-ANSWER-FILTER
+// CFG: PACKAGE-TAG-REQUEST-NODE-LIST-ANSWER-FILTER-CLASS
 $cfg->setConfigEntry('package_tag_request_node_list_answer_filter_class', 'Org\Shipsimu\Hub\Filter\Tag\Network\Answer\Request\NodeList\PackageRequestNodeListAnswerTagFilter');
 
-// CFG: PACKAGE-TAG-DHT-BOOTSTRAP-FILTER
+// CFG: PACKAGE-TAG-DHT-BOOTSTRAP-FILTER-CLASS
 $cfg->setConfigEntry('package_tag_dht_bootstrap_filter_class', 'Org\Shipsimu\Hub\Filter\Tag\Network\Dht\Bootstrap\PackageDhtBootstrapTagFilter');
 
 // CFG: MESSAGE-TYPE-ANNOUNCEMENT-HANDLER-CLASS
@@ -959,28 +959,28 @@ $cfg->setConfigEntry('miner_buffer_stacker_class', 'Org\Mxchange\CoreFramework\S
 // CFG: HUB-CONSOLE-CMD-CRUNCHER-RESOLVER-CLASS
 $cfg->setConfigEntry('hub_console_cmd_miner_resolver_class', 'Org\Shipsimu\Hub\Resolver\Command\HubConsoleCommandResolver');
 
-// CFG: MINER-PHP-REQUIREMENTS-FILTER
+// CFG: MINER-PHP-REQUIREMENTS-FILTER-CLASS
 $cfg->setConfigEntry('miner_php_requirements_filter_class', 'Org\Shipsimu\Hub\Miner\Filter\Requirements\MinerPhpRequirementsFilter');
 
-// CFG: MINER-INITIALIZER-FILTER
+// CFG: MINER-INITIALIZER-FILTER-CLASS
 $cfg->setConfigEntry('miner_initializer_filter_class', 'Org\Shipsimu\Hub\Miner\Filter\Initialization\MinerInitializationFilter');
 
-// CFG: MINER-WELCOME-TEASER-FILTER
+// CFG: MINER-WELCOME-TEASER-FILTER-CLASS
 $cfg->setConfigEntry('miner_welcome_teaser_filter_class', 'Org\Shipsimu\Hub\Miner\Filter\WelcomeTeaser\MinerWelcomeTeaserFilter');
 
-// CFG: MINER-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER
+// CFG: MINER-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER-CLASS
 $cfg->setConfigEntry('miner_bootstrap_task_handler_initializer_filter_class', 'Org\Shipsimu\Hub\Filter\Initializer\Miner\Handler\Task\MinerTaskHandlerInitializerFilter');
 
-// CFG: CRUNCHER-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER
+// CFG: CRUNCHER-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER-CLASS
 $cfg->setConfigEntry('miner_bootstrap_extra_bootstrapping_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Miner\Extra\MinerBootstrapExtraBootstrappingFilter');
 
-// CFG: MINHER-BOOTSTRAP-BUFFER-QUEUE-INITIALIZER-FILTER
+// CFG: MINHER-BOOTSTRAP-BUFFER-QUEUE-INITIALIZER-FILTER-CLASS
 $cfg->setConfigEntry('miner_bootstrap_buffer_queue_initializer_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Miner\Queue\MinerBootstrapBufferQueueInitializerFilter');
 
-// CFG: MINER-BOOTSTRAP-GENERIC-ACTIVATION-FILTER
+// CFG: MINER-BOOTSTRAP-GENERIC-ACTIVATION-FILTER-CLASS
 $cfg->setConfigEntry('miner_bootstrap_generic_activation_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Miner\Activation\MinerBootstrapGenericActivationFilter');
 
-// CFG: MINER-SHUTDOWN-FILTER
+// CFG: MINER-SHUTDOWN-FILTER-CLASS
 $cfg->setConfigEntry('miner_shutdown_filter_class', 'Org\Shipsimu\Hub\Filter\Shutdown\Miner\MinerShutdownFilter');
 
 // CFG: MINER-INIT-STATE-CLASS
@@ -1071,28 +1071,28 @@ $cfg->setConfigEntry('cruncher_buffer_stacker_class', 'Org\Mxchange\CoreFramewor
 // CFG: HUB-CONSOLE-CMD-CRUNCHER-RESOLVER-CLASS
 $cfg->setConfigEntry('hub_console_cmd_cruncher_resolver_class', 'Org\Shipsimu\Hub\Resolver\Command\HubConsoleCommandResolver');
 
-// CFG: CRUNCHER-PHP-REQUIREMENTS-FILTER
+// CFG: CRUNCHER-PHP-REQUIREMENTS-FILTER-CLASS
 $cfg->setConfigEntry('cruncher_php_requirements_filter_class', 'Org\Shipsimu\Hub\Cruncher\Filter\Requirements\CruncherPhpRequirementsFilter');
 
-// CFG: CRUNCHER-INITIALIZER-FILTER
+// CFG: CRUNCHER-INITIALIZER-FILTER-CLASS
 $cfg->setConfigEntry('cruncher_initializer_filter_class', 'Org\Shipsimu\Hub\Cruncher\Filter\Initialization\CruncherInitializationFilter');
 
-// CFG: CRUNCHER-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER
+// CFG: CRUNCHER-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER-CLASS
 $cfg->setConfigEntry('cruncher_bootstrap_task_handler_initializer_filter_class', 'Org\Shipsimu\Hub\Filter\Initializer\Cruncher\Handler\Task\CruncherTaskHandlerInitializerFilter');
 
-// CFG: CRUNCHER-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER
+// CFG: CRUNCHER-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER-CLASS
 $cfg->setConfigEntry('cruncher_bootstrap_extra_bootstrapping_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Cruncher\Extra\CruncherBootstrapExtraBootstrappingFilter');
 
-// CFG: CRUNCHER-BOOTSTRAP-BUFFER-QUEUE-INITIALIZER-FILTER
+// CFG: CRUNCHER-BOOTSTRAP-BUFFER-QUEUE-INITIALIZER-FILTER-CLASS
 $cfg->setConfigEntry('cruncher_bootstrap_buffer_queue_initializer_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Cruncher\Queue\CruncherBootstrapBufferQueueInitializerFilter');
 
-// CFG: CRUNCHER-BOOTSTRAP-GENERIC-ACTIVATION-FILTER
+// CFG: CRUNCHER-BOOTSTRAP-GENERIC-ACTIVATION-FILTER-CLASS
 $cfg->setConfigEntry('cruncher_bootstrap_generic_activation_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Cruncher\Activation\CruncherBootstrapGenericActivationFilter');
 
-// CFG: CRUNCHER-WELCOME-TEASER-FILTER
+// CFG: CRUNCHER-WELCOME-TEASER-FILTER-CLASS
 $cfg->setConfigEntry('cruncher_welcome_teaser_filter_class', 'Org\Shipsimu\Hub\Cruncher\Filter\WelcomeTeaser\CruncherWelcomeTeaserFilter');
 
-// CFG: CRUNCHER-SHUTDOWN-FILTER
+// CFG: CRUNCHER-SHUTDOWN-FILTER-CLASS
 $cfg->setConfigEntry('cruncher_shutdown_filter_class', 'Org\Shipsimu\Hub\Filter\Shutdown\Cruncher\CruncherShutdownFilter');
 
 // CFG: CRUNCHER-WORK-UNIT-FETCHER-TASK-CLASS
@@ -1210,28 +1210,28 @@ $cfg->setConfigEntry('cruncher_per_unit_key_limit', 1000);
 // CFG: HUB-CONSOLE-CMD-CHAT-RESOLVER-CLASS
 $cfg->setConfigEntry('hub_console_cmd_chat_resolver_class', 'Org\Shipsimu\Hub\Resolver\Command\HubConsoleCommandResolver');
 
-// CFG: CHAT-PHP-REQUIREMENTS-FILTER
+// CFG: CHAT-PHP-REQUIREMENTS-FILTER-CLASS
 $cfg->setConfigEntry('chat_php_requirements_filter_class', 'ChatPhpRequirementsFilter');
 
-// CFG: CHAT-INITIALIZER-FILTER
+// CFG: CHAT-INITIALIZER-FILTER-CLASS
 $cfg->setConfigEntry('chat_initializer_filter_class', 'ChatInitializationFilter');
 
 // CFG: NODE-CONSOLE-CHAT
 $cfg->setConfigEntry('node_console_chat_class', 'NodeConsoleChat');
 
-// CFG: CHAT-WELCOME-TEASER-FILTER
+// CFG: CHAT-WELCOME-TEASER-FILTER-CLASS
 $cfg->setConfigEntry('chat_welcome_teaser_filter_class', 'ChatWelcomeTeaserFilter');
 
-// CFG: CHAT-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER
+// CFG: CHAT-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER-CLASS
 $cfg->setConfigEntry('chat_bootstrap_task_handler_initializer_filter_class', 'ChatTaskHandlerInitializerFilter');
 
-// CFG: CHAT-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER
+// CFG: CHAT-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER-CLASS
 $cfg->setConfigEntry('chat_bootstrap_extra_bootstrapping_filter_class', 'ChatBootstrapExtraBootstrappingFilter');
 
-// CFG: CHAT-BOOTSTRAP-GENERIC-ACTIVATION-FILTER
+// CFG: CHAT-BOOTSTRAP-GENERIC-ACTIVATION-FILTER-CLASS
 $cfg->setConfigEntry('chat_bootstrap_generic_activation_filter_class', 'ChatBootstrapGenericActivationFilter');
 
-// CFG: CHAT-SHUTDOWN-FILTER
+// CFG: CHAT-SHUTDOWN-FILTER-CLASS
 $cfg->setConfigEntry('chat_shutdown_filter_class', 'ChatShutdownFilter');
 
 // CFG: CHAT-TELNET-LISTENER-TASK-CLASS
@@ -1262,28 +1262,28 @@ $cfg->setConfigEntry('index_extension', '.idx');
 // CFG: HUB-CONSOLE-CMD-CHAT-RESOLVER-CLASS
 $cfg->setConfigEntry('hub_console_cmd_crawler_resolver_class', 'Org\Shipsimu\Hub\Resolver\Command\HubConsoleCommandResolver');
 
-// CFG: CRAWLER-PHP-REQUIREMENTS-FILTER
+// CFG: CRAWLER-PHP-REQUIREMENTS-FILTER-CLASS
 $cfg->setConfigEntry('crawler_php_requirements_filter_class', 'Org\Shipsimu\Hub\Crawler\Filter\Requirements\CrawlerPhpRequirementsFilter');
 
-// CFG: CRAWLER-INITIALIZER-FILTER
+// CFG: CRAWLER-INITIALIZER-FILTER-CLASS
 $cfg->setConfigEntry('crawler_initializer_filter_class', 'Org\Shipsimu\Hub\Crawler\Filter\Initialization\CrawlerInitializationFilter');
 
-// CFG: NODE-CONSOLE-CRAWLER
-$cfg->setConfigEntry('node_console_crawler_class', 'Org\Shipsimu\Hub\Crawler\NodeConsoleCrawler');
+// CFG: NODE-CONSOLE-DISTRIBUTED-CRAWLER-CLASS
+$cfg->setConfigEntry('node_console_distributed_crawler_class', 'Org\Shipsimu\Hub\Crawler\NodeConsoleDistributedCrawler');
 
-// CFG: CRAWLER-WELCOME-TEASER-FILTER
+// CFG: CRAWLER-WELCOME-TEASER-FILTER-CLASS
 $cfg->setConfigEntry('crawler_welcome_teaser_filter_class', 'Org\Shipsimu\Hub\Crawler\Filter\WelcomeTeaser\CrawlerWelcomeTeaserFilter');
 
-// CFG: CRAWLER-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER
-$cfg->setConfigEntry('crawler_bootstrap_task_handler_initializer_filter_class', 'Org\Shipsimu\Hub\Filter\Initializer\Crawler\Handler\Task\CrawlerTaskHandlerInitializerFilter');
+// CFG: CRAWLER-BOOTSTRAP-DISTRIBUTED-TASK-HANDLER-INITIALIZER-FILTER-CLASS
+$cfg->setConfigEntry('crawler_bootstrap_distributed_task_handler_initializer_filter_class', 'Org\Shipsimu\Hub\Filter\Initializer\Crawler\Handler\Task\CrawlerDistributedTaskHandlerInitializerFilter');
 
-// CFG: CRAWLER-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER
+// CFG: CRAWLER-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER-CLASS
 $cfg->setConfigEntry('crawler_bootstrap_extra_bootstrapping_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Crawler\Extra\CrawlerBootstrapExtraBootstrappingFilter');
 
-// CFG: CRAWLER-BOOTSTRAP-GENERIC-ACTIVATION-FILTER
+// CFG: CRAWLER-BOOTSTRAP-GENERIC-ACTIVATION-FILTER-CLASS
 $cfg->setConfigEntry('crawler_bootstrap_generic_activation_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Crawler\Activation\CrawlerBootstrapGenericActivationFilter');
 
-// CFG: CRAWLER-SHUTDOWN-FILTER
+// CFG: CRAWLER-SHUTDOWN-FILTER-CLASS
 $cfg->setConfigEntry('crawler_shutdown_filter_class', 'Org\Shipsimu\Hub\Filter\Shutdown\Crawler\CrawlerShutdownFilter');
 
 // CFG: CRAWLER-NODE-COMMUNICATOR-CLASS
@@ -1515,28 +1515,28 @@ $cfg->setConfigEntry('crawler_url_list_column_separator', ',');
 // CFG: HUB-CONSOLE-CMD-APT-PROXY-RESOLVER-CLASS
 $cfg->setConfigEntry('hub_console_cmd_apt_proxy_resolver_class', 'Org\Shipsimu\Hub\Resolver\Command\HubConsoleCommandResolver');
 
-// CFG: APT-PROXY-PHP-REQUIREMENTS-FILTER
+// CFG: APT-PROXY-PHP-REQUIREMENTS-FILTER-CLASS
 $cfg->setConfigEntry('apt_proxy_php_requirements_filter_class', 'AptProxyPhpRequirementsFilter');
 
-// CFG: APT-PROXY-INITIALIZER-FILTER
+// CFG: APT-PROXY-INITIALIZER-FILTER-CLASS
 $cfg->setConfigEntry('apt_proxy_initializer_filter_class', 'AptProxyInitializationFilter');
 
 // CFG: NODE-CONSOLE-APT-PROXY
 $cfg->setConfigEntry('node_console_apt_proxy_class', 'NodeConsoleAptProxy');
 
-// CFG: APT-PROXY-WELCOME-TEASER-FILTER
+// CFG: APT-PROXY-WELCOME-TEASER-FILTER-CLASS
 $cfg->setConfigEntry('apt_proxy_welcome_teaser_filter_class', 'AptProxyWelcomeTeaserFilter');
 
-// CFG: APT-PROXY-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER
+// CFG: APT-PROXY-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER-CLASS
 $cfg->setConfigEntry('apt_proxy_bootstrap_task_handler_initializer_filter_class', 'AptProxyTaskHandlerInitializerFilter');
 
-// CFG: APT-PROXY-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER
+// CFG: APT-PROXY-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER-CLASS
 $cfg->setConfigEntry('apt_proxy_bootstrap_extra_bootstrapping_filter_class', 'AptProxyBootstrapExtraBootstrappingFilter');
 
-// CFG: APT-PROXY-BOOTSTRAP-GENERIC-ACTIVATION-FILTER
+// CFG: APT-PROXY-BOOTSTRAP-GENERIC-ACTIVATION-FILTER-CLASS
 $cfg->setConfigEntry('apt_proxy_bootstrap_generic_activation_filter_class', 'AptProxyBootstrapGenericActivationFilter');
 
-// CFG: APT-PROXY-SHUTDOWN-FILTER
+// CFG: APT-PROXY-SHUTDOWN-FILTER-CLASS
 $cfg->setConfigEntry('apt_proxy_shutdown_filter_class', 'AptProxyShutdownFilter');
 
 // CFG: APT-PROXY-LISTENER-TASK-CLASS
diff --git a/core b/core
index 93c10cd8bbd88a27bdcd0e65956298666218f150..53bf33122fffeedc46ea50c84cbbdd9e2aabd45d 160000 (submodule)
--- a/core
+++ b/core
@@ -1 +1 @@
-Subproject commit 93c10cd8bbd88a27bdcd0e65956298666218f150
+Subproject commit 53bf33122fffeedc46ea50c84cbbdd9e2aabd45d