From 8dce42efef730a9d22fb352a67ecc1cddf7dc7ca Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Mon, 4 Jan 2021 03:29:32 +0100 Subject: [PATCH] Continued: - renamed default crawler names to 'distributed' as this is now the crawler's default mode. Later more modes, e.g. 'local' can be added which won't register any remote jobs and maybe not communicate with the node (hub) to avoid crawler info distribution - Just a wild idea! - updated core framework MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Signed-off-by: Roland Häder --- .../class_HubConsoleCrawlerCommand.php | 1 - ...> class_NodeConsoleDistributedCrawler.php} | 12 +- .../classes/filter/class_BaseHubFilter.php | 4 +- .../class_CrawlerInitializationFilter.php | 13 +- .../task/class_BaseCrawlerTaskFilter.php | 85 ++++++++++++ ...stributedTaskHandlerInitializerFilter.php} | 41 ++---- .../class_CrawlerUploadedListUrlSource.php | 4 +- application/hub/config.php | 126 +++++++++--------- core | 2 +- 9 files changed, 182 insertions(+), 106 deletions(-) rename application/hub/classes/crawler/console/{class_NodeConsoleCrawler.php => class_NodeConsoleDistributedCrawler.php} (86%) create mode 100644 application/hub/classes/filter/task/class_BaseCrawlerTaskFilter.php rename application/hub/classes/filter/task/crawler/{class_CrawlerTaskHandlerInitializerFilter.php => class_CrawlerDistributedTaskHandlerInitializerFilter.php} (71%) diff --git a/application/hub/classes/commands/console/class_HubConsoleCrawlerCommand.php b/application/hub/classes/commands/console/class_HubConsoleCrawlerCommand.php index 6b02bc3b5..eb9df8db3 100644 --- a/application/hub/classes/commands/console/class_HubConsoleCrawlerCommand.php +++ b/application/hub/classes/commands/console/class_HubConsoleCrawlerCommand.php @@ -127,7 +127,6 @@ class HubConsoleCrawlerCommand extends BaseCommand implements Commandable { $controllerInstance->addPreFilter(ObjectFactory::createObjectByConfiguredName('crawler_welcome_teaser_filter_class')); // Add bootstrap filters - $controllerInstance->addBootstrapFilter(ObjectFactory::createObjectByConfiguredName('crawler_bootstrap_task_handler_initializer_filter_class')); $controllerInstance->addBootstrapFilter(ObjectFactory::createObjectByConfiguredName('crawler_bootstrap_extra_bootstrapping_filter_class')); // This is the last generic boostrap filter diff --git a/application/hub/classes/crawler/console/class_NodeConsoleCrawler.php b/application/hub/classes/crawler/console/class_NodeConsoleDistributedCrawler.php similarity index 86% rename from application/hub/classes/crawler/console/class_NodeConsoleCrawler.php rename to application/hub/classes/crawler/console/class_NodeConsoleDistributedCrawler.php index d834b36ae..d23b25734 100644 --- a/application/hub/classes/crawler/console/class_NodeConsoleCrawler.php +++ b/application/hub/classes/crawler/console/class_NodeConsoleDistributedCrawler.php @@ -4,13 +4,14 @@ namespace Org\Shipsimu\Hub\Crawler; // Import framework stuff use Org\Mxchange\CoreFramework\Controller\Controller; +use Org\Mxchange\CoreFramework\Factory\Object\ObjectFactory; use Org\Mxchange\CoreFramework\Helper\Application\ApplicationHelper; use Org\Mxchange\CoreFramework\Registry\GenericRegistry; use Org\Mxchange\CoreFramework\Registry\Registerable; use Org\Mxchange\CoreFramework\Response\Responseable; /** - * A node console crawler + * A distributed node console crawler * * @author Roland Haeder * @version 0.0.0 @@ -31,7 +32,7 @@ use Org\Mxchange\CoreFramework\Response\Responseable; * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -class NodeConsoleCrawler extends BaseNodeCrawler implements Crawler, Registerable { +class NodeConsoleDistributedCrawler extends BaseNodeCrawler implements Crawler, Registerable { /** * Protected constructor * @@ -47,9 +48,9 @@ class NodeConsoleCrawler extends BaseNodeCrawler implements Crawler, Registerabl * * @return $crawlerInstance An instance of a Crawler class */ - public final static function createNodeConsoleCrawler () { + public final static function createNodeConsoleDistributedCrawler () { // Get new instance - $crawlerInstance = new NodeConsoleCrawler(); + $crawlerInstance = new NodeConsoleDistributedCrawler(); // Return the prepared instance return $crawlerInstance; @@ -95,7 +96,8 @@ class NodeConsoleCrawler extends BaseNodeCrawler implements Crawler, Registerabl * @return void */ public function addExtraFilters (Controller $controllerInstance, Responseable $responseInstance) { - $this->partialStub('Please implement this method.'); + // Add distributed crawler related bootstrap-filters + $controllerInstance->addBootstrapFilter(ObjectFactory::createObjectByConfiguredName('crawler_bootstrap_distributed_task_handler_initializer_filter_class')); } } diff --git a/application/hub/classes/filter/class_BaseHubFilter.php b/application/hub/classes/filter/class_BaseHubFilter.php index fbf5e822a..d31b697be 100644 --- a/application/hub/classes/filter/class_BaseHubFilter.php +++ b/application/hub/classes/filter/class_BaseHubFilter.php @@ -110,7 +110,7 @@ abstract class BaseHubFilter extends BaseFilter { } // Construct an array for pushing it on next stack - $messageArray = array( + $messageArray = [ // Message data itself NetworkPackageHandler::MESSAGE_ARRAY_DATA => $this->dataXmlNodes, // Message type (which is $messageType) @@ -121,7 +121,7 @@ abstract class BaseHubFilter extends BaseFilter { NetworkPackageHandler::MESSAGE_ARRAY_HASH => $messageData[NetworkPackageHandler::PACKAGE_CONTENT_HASH], // Package tags NetworkPackageHandler::MESSAGE_ARRAY_TAGS => $messageData[NetworkPackageHandler::PACKAGE_CONTENT_TAGS], - ); + ]; // Push the processed message back on stack $handlerInstance->getStackInstance()->pushNamed(NetworkPackageHandler::STACKER_NAME_PROCESSED_MESSAGE, $messageArray); diff --git a/application/hub/classes/filter/crawler/class_CrawlerInitializationFilter.php b/application/hub/classes/filter/crawler/class_CrawlerInitializationFilter.php index 3f1f6ac5b..fee0f05fe 100644 --- a/application/hub/classes/filter/crawler/class_CrawlerInitializationFilter.php +++ b/application/hub/classes/filter/crawler/class_CrawlerInitializationFilter.php @@ -14,7 +14,7 @@ use Org\Mxchange\CoreFramework\Request\Requestable; use Org\Mxchange\CoreFramework\Response\Responseable; /** - * A Initialization filter for crawler + * An iitialization filter for crawlers * * @author Roland Haeder * @version 0.0.0 @@ -68,8 +68,17 @@ class CrawlerInitializationFilter extends BaseCrawlerFilter implements Filterabl * @todo 0% done */ public function execute (Requestable $requestInstance, Responseable $responseInstance) { + // Default crawler mode is full-blown flag ship! + $crawlerMode = 'distributed'; + + // Is 'mode' set? + if ($requestInstance->isRequestElementSet('mode')) { + // Get 'mode' parameter + $crawlerMode = $requestInstance->getRequestElement('mode'); + } + // Get an instance - $crawlerInstance = ObjectFactory::createObjectByConfiguredName('node_console_crawler_class'); + $crawlerInstance = ObjectFactory::createObjectByConfiguredName(sprintf('node_console_%s_crawler_class', $crawlerMode)); // Get a registry $applicationInstance = ApplicationHelper::getSelfInstance(); diff --git a/application/hub/classes/filter/task/class_BaseCrawlerTaskFilter.php b/application/hub/classes/filter/task/class_BaseCrawlerTaskFilter.php new file mode 100644 index 000000000..cbb53b6de --- /dev/null +++ b/application/hub/classes/filter/task/class_BaseCrawlerTaskFilter.php @@ -0,0 +1,85 @@ + + * @version 0.0.0 + * @copyright Copyright (c) 2014 - 2020 Crawler Developer Team + * @license GNU GPL 3.0 or any newer version + * @link http://www.shipsimu.org + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +abstract class BaseCrawlerTaskFilter extends BaseCrawlerFilter { + /** + * Protected constructor + * + * @param $className Real name of class + * @return void + */ + protected function __construct (string $className) { + // Call parent constructor + parent::__construct($className); + } + + /** + * Executes the filter with given request and response objects + * + * @param $requestInstance An instance of a class with an Requestable interface + * @param $responseInstance An instance of a class with an Responseable interface + * @return void + * @throws FilterChainException If the filter chain needs to be interrupted + * @todo 10% done + */ + public function execute (Requestable $requestInstance, Responseable $responseInstance) { + // Get a new task handler instance + $handlerInstance = ObjectFactory::createObjectByConfiguredName('task_handler_class'); + + /* + * Register tasks for all crawlers: + * + * 1) MIME sniffer + */ + $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_mime_sniffer_task_class'); + $handlerInstance->registerTask('crawler_mime_sniffer', $taskInstance); + + // 2) Document parser (converts document to meta format) + $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_document_parser_task_class'); + $handlerInstance->registerTask('crawler_document_parser', $taskInstance); + + // 3) Document structure analyzer + $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_structure_analyzer_task_class'); + $handlerInstance->registerTask('crawler_structure_analyzer', $taskInstance); + + // 4) Snippet extractor + $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_snippet_extractor_task_class'); + $handlerInstance->registerTask('crawler_snippet_extractor', $taskInstance); + + // Add handler instance to registry + GenericRegistry::getRegistry()->addInstance('task_handler', $handlerInstance); + } + +} diff --git a/application/hub/classes/filter/task/crawler/class_CrawlerTaskHandlerInitializerFilter.php b/application/hub/classes/filter/task/crawler/class_CrawlerDistributedTaskHandlerInitializerFilter.php similarity index 71% rename from application/hub/classes/filter/task/crawler/class_CrawlerTaskHandlerInitializerFilter.php rename to application/hub/classes/filter/task/crawler/class_CrawlerDistributedTaskHandlerInitializerFilter.php index 8a264dfcc..a9d1d9976 100644 --- a/application/hub/classes/filter/task/crawler/class_CrawlerTaskHandlerInitializerFilter.php +++ b/application/hub/classes/filter/task/crawler/class_CrawlerDistributedTaskHandlerInitializerFilter.php @@ -3,7 +3,7 @@ namespace Org\Shipsimu\Hub\Filter\Initializer\Crawler\Handler\Task; // Import application-specific stuff -use Org\Shipsimu\Hub\Crawler\Filter\BaseCrawlerFilter; +use Org\Shipsimu\Hub\Crawler\Filter\Task\BaseCrawlerTaskFilter; // Import framework stuff use Org\Mxchange\CoreFramework\Bootstrap\FrameworkBootstrap; @@ -35,7 +35,7 @@ use Org\Mxchange\CoreFramework\Response\Responseable; * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -class CrawlerTaskHandlerInitializerFilter extends BaseCrawlerFilter implements Filterable { +class CrawlerDistributedTaskHandlerInitializerFilter extends BaseCrawlerTaskFilter implements Filterable { /** * Protected constructor * @@ -51,9 +51,9 @@ class CrawlerTaskHandlerInitializerFilter extends BaseCrawlerFilter implements F * * @return $filterInstance An instance of this filter class */ - public static final function createCrawlerTaskHandlerInitializerFilter () { + public static final function createCrawlerDistributedTaskHandlerInitializerFilter () { // Get a new instance - $filterInstance = new CrawlerTaskHandlerInitializerFilter(); + $filterInstance = new CrawlerDistributedTaskHandlerInitializerFilter(); // Return the instance return $filterInstance; @@ -69,14 +69,14 @@ class CrawlerTaskHandlerInitializerFilter extends BaseCrawlerFilter implements F * @todo 10% done */ public function execute (Requestable $requestInstance, Responseable $responseInstance) { - // Get crawler instance - $crawlerInstance = GenericRegistry::getRegistry()->getInstance('crawler'); + // Run parent method first + parent::execute($requestInstance, $responseInstance); - // Get a new task handler instance - $handlerInstance = ObjectFactory::createObjectByConfiguredName('task_handler_class'); + // Get task handler instance + $handlerInstance = GenericRegistry::getRegistry()->getInstance('task_handler'); /* - * Register all tasks: + * Register all tasks for a distributed crawler: * * 1) "Communicator" for crawler->node communication */ @@ -95,27 +95,11 @@ class CrawlerTaskHandlerInitializerFilter extends BaseCrawlerFilter implements F $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_remote_job_publisher_task_class'); $handlerInstance->registerTask('crawler_remote_job_publisher', $taskInstance); - // 5) MIME sniffer - $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_mime_sniffer_task_class'); - $handlerInstance->registerTask('crawler_mime_sniffer', $taskInstance); - - // 6) Document parser (converts document to meta format) - $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_document_parser_task_class'); - $handlerInstance->registerTask('crawler_document_parser', $taskInstance); - - // 7) Document structure analyzer - $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_structure_analyzer_task_class'); - $handlerInstance->registerTask('crawler_structure_analyzer', $taskInstance); - - // 8) Snippet extractor - $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_snippet_extractor_task_class'); - $handlerInstance->registerTask('crawler_snippet_extractor', $taskInstance); - - // 9) Node ping + // 5) Node ping $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_ping_task_class'); $handlerInstance->registerTask('crawler_ping', $taskInstance); - // 10) URL sources + // 6) URL sources foreach (explode(':', FrameworkBootstrap::getConfigurationInstance()->getConfigEntry('crawler_url_stacks')) as $stack) { // Init task instance $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_url_source_' . $stack . '_task_class'); @@ -123,9 +107,6 @@ class CrawlerTaskHandlerInitializerFilter extends BaseCrawlerFilter implements F // And register it $handlerInstance->registerTask('crawler_url_source_' . $stack, $taskInstance); } - - // Put the task handler in registry - GenericRegistry::getRegistry()->addInstance('task_handler', $handlerInstance); } } diff --git a/application/hub/classes/source/urls/class_CrawlerUploadedListUrlSource.php b/application/hub/classes/source/urls/class_CrawlerUploadedListUrlSource.php index 9830b7d8b..f012e66fc 100644 --- a/application/hub/classes/source/urls/class_CrawlerUploadedListUrlSource.php +++ b/application/hub/classes/source/urls/class_CrawlerUploadedListUrlSource.php @@ -164,7 +164,7 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R // Read next entry /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('CRAWLER-UPLOADED-LIST-URL-SOURCE: this->csvFileImported=%s', print_r($this->csvFileImported, TRUE))); - $directoryEntry = $this->getDirectoryInstance()->readDirectoryExcept(array_merge(['.htaccess', '.', '..'], $this->csvFileImported)); + $directoryEntry = $this->getDirectoryInstance()->readDirectoryExcept(array_merge(['.gitkeep', '.', '..'], $this->csvFileImported)); // Is it empty or wrong file extension? /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('CRAWLER-UPLOADED-LIST-URL-SOURCE: directoryEntry(%d)=%s', strlen($directoryEntry), $directoryEntry)); @@ -206,7 +206,7 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R $csvArray = [ UrlSource::URL_SOURCE_ARRAY_START_URL => $csvData[0], UrlSource::URL_SOURCE_ARRAY_START_DEPTH => $csvData[1], - UrlSource::URL_SOURCE_ARRAY_EXTERNAL_DEPTH => $csvData[2] + UrlSource::URL_SOURCE_ARRAY_EXTERNAL_DEPTH => $csvData[2], ]; // Then add more data to it diff --git a/application/hub/config.php b/application/hub/config.php index 45cf27b16..80511a03c 100644 --- a/application/hub/config.php +++ b/application/hub/config.php @@ -171,73 +171,73 @@ $cfg->setConfigEntry('hub_html_controller_resolver_class', 'Org\Shipsimu\Hub\Res // CFG: HUB-CONSOLE-CMD-MAIN-RESOLVER-CLASS $cfg->setConfigEntry('hub_console_cmd_main_resolver_class', 'Org\Shipsimu\Hub\Resolver\Command\HubConsoleCommandResolver'); -// CFG: NEWS-DOWNLOAD-FILTER +// CFG: NEWS-DOWNLOAD-FILTER-CLASS $cfg->setConfigEntry('news_download_filter_class', 'Org\Mxchange\CoreFramework\Filter\News\NewsDownloadFilter'); -// CFG: NEWS-PROCESS-FILTER +// CFG: NEWS-PROCESS-FILTER-CLASS $cfg->setConfigEntry('news_process_filter_class', 'Org\Mxchange\CoreFramework\Filter\News\NewsProcessFilter'); -// CFG: NODE-PHP-REQUIREMENTS-FILTER +// CFG: NODE-PHP-REQUIREMENTS-FILTER-CLASS $cfg->setConfigEntry('node_php_requirements_filter_class', 'Org\Shipsimu\Hub\Filter\Node\Requirements\NodePhpRequirementsFilter'); -// CFG: NODE-INITIALIZER-FILTER +// CFG: NODE-INITIALIZER-FILTER-CLASS $cfg->setConfigEntry('node_initializer_filter_class', 'Org\Shipsimu\Hub\Filter\Node\Initialization\NodeInitializationFilter'); -// CFG: NODE-ACTIVATION-TASK-HANDLER-INITIALIZER-FILTER +// CFG: NODE-ACTIVATION-TASK-HANDLER-INITIALIZER-FILTER-CLASS $cfg->setConfigEntry('node_activation_task_handler_initializer_filter_class', 'Org\Shipsimu\Hub\Filter\Initializer\Node\Handler\Task\NodeTaskHandlerInitializerFilter'); -// CFG: NODE-ACTIVATION-ANNOUNCEMENT-TASK-FILTER +// CFG: NODE-ACTIVATION-ANNOUNCEMENT-TASK-FILTER-CLASS $cfg->setConfigEntry('node_activation_announcement_task_filter_class', 'Org\Shipsimu\Hub\Filter\Activation\Node\Announcement\NodeActivationAnnouncementFilter'); -// CFG: NODE-WELCOME-TEASER-FILTER +// CFG: NODE-WELCOME-TEASER-FILTER-CLASS $cfg->setConfigEntry('node_welcome_teaser_filter_class', 'Org\Shipsimu\Hub\Filter\Node\WelcomeTeaser\NodeWelcomeTeaserFilter'); -// CFG: NODE-BOOTSTRAP-GENERATE-NODEID-FILTER +// CFG: NODE-BOOTSTRAP-GENERATE-NODEID-FILTER-CLASS $cfg->setConfigEntry('node_bootstrap_generate_nodeid_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Node\NodeId\NodeBootstrapGenerateNodeIdFilter'); -// CFG: NODE-BOOTSTRAP-GENERATE-SESSIONID-FILTER +// CFG: NODE-BOOTSTRAP-GENERATE-SESSIONID-FILTER-CLASS $cfg->setConfigEntry('node_bootstrap_generate_sessionid_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Node\SessionId\NodeBootstrapGenerateSessionIdFilter'); -// CFG: NODE-BOOTSTRAP-GENERATE-PRIVATE-KEY-FILTER +// CFG: NODE-BOOTSTRAP-GENERATE-PRIVATE-KEY-FILTER-CLASS $cfg->setConfigEntry('node_bootstrap_generate_private_key_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Node\PrivateKey\NodeBootstrapGeneratePrivateKeyFilter'); -// CFG: NODE-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER +// CFG: NODE-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER-CLASS $cfg->setConfigEntry('node_bootstrap_extra_bootstrapping_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Node\Extra\NodeBootstrapExtraBootstrappingFilter'); -// CFG: NODE-BOOTSTRAP-LISTENER-POOL-FILTER +// CFG: NODE-BOOTSTRAP-LISTENER-POOL-FILTER-CLASS $cfg->setConfigEntry('node_bootstrap_listener_pool_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Node\Pool\NodeBootstrapListenerPoolFilter'); -// CFG: NODE-SHUTDOWN-FLUSH-NODE-LIST-FILTER +// CFG: NODE-SHUTDOWN-FLUSH-NODE-LIST-FILTER-CLASS $cfg->setConfigEntry('node_shutdown_flush_node_list_filter_class', 'Org\Shipsimu\Hub\Filter\Shutdown\Node\NodeList\NodeShutdownFlushNodeListFilter'); -// CFG: NODE-SHUTDOWN-TASK-HANDLER-FILTER +// CFG: NODE-SHUTDOWN-TASK-HANDLER-FILTER-CLASS $cfg->setConfigEntry('node_shutdown_task_handler_filter_class', 'Org\Shipsimu\Hub\Filter\Shutdown\Node\Handler\Task\NodeShutdownTaskHandlerFilter'); -// CFG: NODE-SHUTDOWN-NODE-FILTER +// CFG: NODE-SHUTDOWN-NODE-FILTER-CLASS $cfg->setConfigEntry('node_shutdown_node_filter_class', 'Org\Shipsimu\Hub\Filter\Shutdown\Node\NodeShutdownNodeFilter'); -// CFG: PACKAGE-FILTER-CHAIN-CLASS +// CFG: PACKAGE-FILTER-CLASS-CHAIN-CLASS $cfg->setConfigEntry('package_filter_chain_class', 'Org\Shipsimu\Hub\FilterChain\Package\PackageFilterChain'); -// CFG: PACKAGE-TAG-ANNOUNCEMENT-FILTER +// CFG: PACKAGE-TAG-ANNOUNCEMENT-FILTER-CLASS $cfg->setConfigEntry('package_tag_announcement_filter_class', 'Org\Shipsimu\Hub\Filter\Tag\Network\Announcement\PackageAnnouncementTagFilter'); -// CFG: PACKAGE-TAG-ANNOUNCEMENT-ANSWER-FILTER +// CFG: PACKAGE-TAG-ANNOUNCEMENT-ANSWER-FILTER-CLASS $cfg->setConfigEntry('package_tag_announcement_answer_filter_class', 'Org\Shipsimu\Hub\Filter\Tag\Network\Answer\Announcement\PackageAnnouncementAnswerTagFilter'); -// CFG: PACKAGE-TAG-DHT-BOOTSTRAP-ANSWER-FILTER +// CFG: PACKAGE-TAG-DHT-BOOTSTRAP-ANSWER-FILTER-CLASS $cfg->setConfigEntry('package_tag_dht_bootstrap_answer_filter_class', 'Org\Shipsimu\Hub\Filter\Tag\Network\Answer\Dht\Bootstrap\PackageDhtBootstrapAnswerTagFilter'); -// CFG: PACKAGE-TAG-SELF-CONNECT-FILTER +// CFG: PACKAGE-TAG-SELF-CONNECT-FILTER-CLASS $cfg->setConfigEntry('package_tag_self_connect_filter_class', 'Org\Shipsimu\Hub\Filter\Tag\Network\SelfConnect\PackageSelfConnectTagFilter'); -// CFG: PACKAGE-TAG-REQUEST-NODE-LIST-FILTER +// CFG: PACKAGE-TAG-REQUEST-NODE-LIST-FILTER-CLASS $cfg->setConfigEntry('package_tag_request_node_list_filter_class', 'Org\Shipsimu\Hub\Filter\Tag\Network\Request\NodeList\PackageRequestNodeListTagFilter'); -// CFG: PACKAGE-TAG-REQUEST-NODE-LIST-ANSWER-FILTER +// CFG: PACKAGE-TAG-REQUEST-NODE-LIST-ANSWER-FILTER-CLASS $cfg->setConfigEntry('package_tag_request_node_list_answer_filter_class', 'Org\Shipsimu\Hub\Filter\Tag\Network\Answer\Request\NodeList\PackageRequestNodeListAnswerTagFilter'); -// CFG: PACKAGE-TAG-DHT-BOOTSTRAP-FILTER +// CFG: PACKAGE-TAG-DHT-BOOTSTRAP-FILTER-CLASS $cfg->setConfigEntry('package_tag_dht_bootstrap_filter_class', 'Org\Shipsimu\Hub\Filter\Tag\Network\Dht\Bootstrap\PackageDhtBootstrapTagFilter'); // CFG: MESSAGE-TYPE-ANNOUNCEMENT-HANDLER-CLASS @@ -959,28 +959,28 @@ $cfg->setConfigEntry('miner_buffer_stacker_class', 'Org\Mxchange\CoreFramework\S // CFG: HUB-CONSOLE-CMD-CRUNCHER-RESOLVER-CLASS $cfg->setConfigEntry('hub_console_cmd_miner_resolver_class', 'Org\Shipsimu\Hub\Resolver\Command\HubConsoleCommandResolver'); -// CFG: MINER-PHP-REQUIREMENTS-FILTER +// CFG: MINER-PHP-REQUIREMENTS-FILTER-CLASS $cfg->setConfigEntry('miner_php_requirements_filter_class', 'Org\Shipsimu\Hub\Miner\Filter\Requirements\MinerPhpRequirementsFilter'); -// CFG: MINER-INITIALIZER-FILTER +// CFG: MINER-INITIALIZER-FILTER-CLASS $cfg->setConfigEntry('miner_initializer_filter_class', 'Org\Shipsimu\Hub\Miner\Filter\Initialization\MinerInitializationFilter'); -// CFG: MINER-WELCOME-TEASER-FILTER +// CFG: MINER-WELCOME-TEASER-FILTER-CLASS $cfg->setConfigEntry('miner_welcome_teaser_filter_class', 'Org\Shipsimu\Hub\Miner\Filter\WelcomeTeaser\MinerWelcomeTeaserFilter'); -// CFG: MINER-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER +// CFG: MINER-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER-CLASS $cfg->setConfigEntry('miner_bootstrap_task_handler_initializer_filter_class', 'Org\Shipsimu\Hub\Filter\Initializer\Miner\Handler\Task\MinerTaskHandlerInitializerFilter'); -// CFG: CRUNCHER-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER +// CFG: CRUNCHER-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER-CLASS $cfg->setConfigEntry('miner_bootstrap_extra_bootstrapping_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Miner\Extra\MinerBootstrapExtraBootstrappingFilter'); -// CFG: MINHER-BOOTSTRAP-BUFFER-QUEUE-INITIALIZER-FILTER +// CFG: MINHER-BOOTSTRAP-BUFFER-QUEUE-INITIALIZER-FILTER-CLASS $cfg->setConfigEntry('miner_bootstrap_buffer_queue_initializer_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Miner\Queue\MinerBootstrapBufferQueueInitializerFilter'); -// CFG: MINER-BOOTSTRAP-GENERIC-ACTIVATION-FILTER +// CFG: MINER-BOOTSTRAP-GENERIC-ACTIVATION-FILTER-CLASS $cfg->setConfigEntry('miner_bootstrap_generic_activation_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Miner\Activation\MinerBootstrapGenericActivationFilter'); -// CFG: MINER-SHUTDOWN-FILTER +// CFG: MINER-SHUTDOWN-FILTER-CLASS $cfg->setConfigEntry('miner_shutdown_filter_class', 'Org\Shipsimu\Hub\Filter\Shutdown\Miner\MinerShutdownFilter'); // CFG: MINER-INIT-STATE-CLASS @@ -1071,28 +1071,28 @@ $cfg->setConfigEntry('cruncher_buffer_stacker_class', 'Org\Mxchange\CoreFramewor // CFG: HUB-CONSOLE-CMD-CRUNCHER-RESOLVER-CLASS $cfg->setConfigEntry('hub_console_cmd_cruncher_resolver_class', 'Org\Shipsimu\Hub\Resolver\Command\HubConsoleCommandResolver'); -// CFG: CRUNCHER-PHP-REQUIREMENTS-FILTER +// CFG: CRUNCHER-PHP-REQUIREMENTS-FILTER-CLASS $cfg->setConfigEntry('cruncher_php_requirements_filter_class', 'Org\Shipsimu\Hub\Cruncher\Filter\Requirements\CruncherPhpRequirementsFilter'); -// CFG: CRUNCHER-INITIALIZER-FILTER +// CFG: CRUNCHER-INITIALIZER-FILTER-CLASS $cfg->setConfigEntry('cruncher_initializer_filter_class', 'Org\Shipsimu\Hub\Cruncher\Filter\Initialization\CruncherInitializationFilter'); -// CFG: CRUNCHER-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER +// CFG: CRUNCHER-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER-CLASS $cfg->setConfigEntry('cruncher_bootstrap_task_handler_initializer_filter_class', 'Org\Shipsimu\Hub\Filter\Initializer\Cruncher\Handler\Task\CruncherTaskHandlerInitializerFilter'); -// CFG: CRUNCHER-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER +// CFG: CRUNCHER-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER-CLASS $cfg->setConfigEntry('cruncher_bootstrap_extra_bootstrapping_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Cruncher\Extra\CruncherBootstrapExtraBootstrappingFilter'); -// CFG: CRUNCHER-BOOTSTRAP-BUFFER-QUEUE-INITIALIZER-FILTER +// CFG: CRUNCHER-BOOTSTRAP-BUFFER-QUEUE-INITIALIZER-FILTER-CLASS $cfg->setConfigEntry('cruncher_bootstrap_buffer_queue_initializer_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Cruncher\Queue\CruncherBootstrapBufferQueueInitializerFilter'); -// CFG: CRUNCHER-BOOTSTRAP-GENERIC-ACTIVATION-FILTER +// CFG: CRUNCHER-BOOTSTRAP-GENERIC-ACTIVATION-FILTER-CLASS $cfg->setConfigEntry('cruncher_bootstrap_generic_activation_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Cruncher\Activation\CruncherBootstrapGenericActivationFilter'); -// CFG: CRUNCHER-WELCOME-TEASER-FILTER +// CFG: CRUNCHER-WELCOME-TEASER-FILTER-CLASS $cfg->setConfigEntry('cruncher_welcome_teaser_filter_class', 'Org\Shipsimu\Hub\Cruncher\Filter\WelcomeTeaser\CruncherWelcomeTeaserFilter'); -// CFG: CRUNCHER-SHUTDOWN-FILTER +// CFG: CRUNCHER-SHUTDOWN-FILTER-CLASS $cfg->setConfigEntry('cruncher_shutdown_filter_class', 'Org\Shipsimu\Hub\Filter\Shutdown\Cruncher\CruncherShutdownFilter'); // CFG: CRUNCHER-WORK-UNIT-FETCHER-TASK-CLASS @@ -1210,28 +1210,28 @@ $cfg->setConfigEntry('cruncher_per_unit_key_limit', 1000); // CFG: HUB-CONSOLE-CMD-CHAT-RESOLVER-CLASS $cfg->setConfigEntry('hub_console_cmd_chat_resolver_class', 'Org\Shipsimu\Hub\Resolver\Command\HubConsoleCommandResolver'); -// CFG: CHAT-PHP-REQUIREMENTS-FILTER +// CFG: CHAT-PHP-REQUIREMENTS-FILTER-CLASS $cfg->setConfigEntry('chat_php_requirements_filter_class', 'ChatPhpRequirementsFilter'); -// CFG: CHAT-INITIALIZER-FILTER +// CFG: CHAT-INITIALIZER-FILTER-CLASS $cfg->setConfigEntry('chat_initializer_filter_class', 'ChatInitializationFilter'); // CFG: NODE-CONSOLE-CHAT $cfg->setConfigEntry('node_console_chat_class', 'NodeConsoleChat'); -// CFG: CHAT-WELCOME-TEASER-FILTER +// CFG: CHAT-WELCOME-TEASER-FILTER-CLASS $cfg->setConfigEntry('chat_welcome_teaser_filter_class', 'ChatWelcomeTeaserFilter'); -// CFG: CHAT-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER +// CFG: CHAT-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER-CLASS $cfg->setConfigEntry('chat_bootstrap_task_handler_initializer_filter_class', 'ChatTaskHandlerInitializerFilter'); -// CFG: CHAT-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER +// CFG: CHAT-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER-CLASS $cfg->setConfigEntry('chat_bootstrap_extra_bootstrapping_filter_class', 'ChatBootstrapExtraBootstrappingFilter'); -// CFG: CHAT-BOOTSTRAP-GENERIC-ACTIVATION-FILTER +// CFG: CHAT-BOOTSTRAP-GENERIC-ACTIVATION-FILTER-CLASS $cfg->setConfigEntry('chat_bootstrap_generic_activation_filter_class', 'ChatBootstrapGenericActivationFilter'); -// CFG: CHAT-SHUTDOWN-FILTER +// CFG: CHAT-SHUTDOWN-FILTER-CLASS $cfg->setConfigEntry('chat_shutdown_filter_class', 'ChatShutdownFilter'); // CFG: CHAT-TELNET-LISTENER-TASK-CLASS @@ -1262,28 +1262,28 @@ $cfg->setConfigEntry('index_extension', '.idx'); // CFG: HUB-CONSOLE-CMD-CHAT-RESOLVER-CLASS $cfg->setConfigEntry('hub_console_cmd_crawler_resolver_class', 'Org\Shipsimu\Hub\Resolver\Command\HubConsoleCommandResolver'); -// CFG: CRAWLER-PHP-REQUIREMENTS-FILTER +// CFG: CRAWLER-PHP-REQUIREMENTS-FILTER-CLASS $cfg->setConfigEntry('crawler_php_requirements_filter_class', 'Org\Shipsimu\Hub\Crawler\Filter\Requirements\CrawlerPhpRequirementsFilter'); -// CFG: CRAWLER-INITIALIZER-FILTER +// CFG: CRAWLER-INITIALIZER-FILTER-CLASS $cfg->setConfigEntry('crawler_initializer_filter_class', 'Org\Shipsimu\Hub\Crawler\Filter\Initialization\CrawlerInitializationFilter'); -// CFG: NODE-CONSOLE-CRAWLER -$cfg->setConfigEntry('node_console_crawler_class', 'Org\Shipsimu\Hub\Crawler\NodeConsoleCrawler'); +// CFG: NODE-CONSOLE-DISTRIBUTED-CRAWLER-CLASS +$cfg->setConfigEntry('node_console_distributed_crawler_class', 'Org\Shipsimu\Hub\Crawler\NodeConsoleDistributedCrawler'); -// CFG: CRAWLER-WELCOME-TEASER-FILTER +// CFG: CRAWLER-WELCOME-TEASER-FILTER-CLASS $cfg->setConfigEntry('crawler_welcome_teaser_filter_class', 'Org\Shipsimu\Hub\Crawler\Filter\WelcomeTeaser\CrawlerWelcomeTeaserFilter'); -// CFG: CRAWLER-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER -$cfg->setConfigEntry('crawler_bootstrap_task_handler_initializer_filter_class', 'Org\Shipsimu\Hub\Filter\Initializer\Crawler\Handler\Task\CrawlerTaskHandlerInitializerFilter'); +// CFG: CRAWLER-BOOTSTRAP-DISTRIBUTED-TASK-HANDLER-INITIALIZER-FILTER-CLASS +$cfg->setConfigEntry('crawler_bootstrap_distributed_task_handler_initializer_filter_class', 'Org\Shipsimu\Hub\Filter\Initializer\Crawler\Handler\Task\CrawlerDistributedTaskHandlerInitializerFilter'); -// CFG: CRAWLER-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER +// CFG: CRAWLER-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER-CLASS $cfg->setConfigEntry('crawler_bootstrap_extra_bootstrapping_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Crawler\Extra\CrawlerBootstrapExtraBootstrappingFilter'); -// CFG: CRAWLER-BOOTSTRAP-GENERIC-ACTIVATION-FILTER +// CFG: CRAWLER-BOOTSTRAP-GENERIC-ACTIVATION-FILTER-CLASS $cfg->setConfigEntry('crawler_bootstrap_generic_activation_filter_class', 'Org\Shipsimu\Hub\Filter\Bootstrap\Crawler\Activation\CrawlerBootstrapGenericActivationFilter'); -// CFG: CRAWLER-SHUTDOWN-FILTER +// CFG: CRAWLER-SHUTDOWN-FILTER-CLASS $cfg->setConfigEntry('crawler_shutdown_filter_class', 'Org\Shipsimu\Hub\Filter\Shutdown\Crawler\CrawlerShutdownFilter'); // CFG: CRAWLER-NODE-COMMUNICATOR-CLASS @@ -1515,28 +1515,28 @@ $cfg->setConfigEntry('crawler_url_list_column_separator', ','); // CFG: HUB-CONSOLE-CMD-APT-PROXY-RESOLVER-CLASS $cfg->setConfigEntry('hub_console_cmd_apt_proxy_resolver_class', 'Org\Shipsimu\Hub\Resolver\Command\HubConsoleCommandResolver'); -// CFG: APT-PROXY-PHP-REQUIREMENTS-FILTER +// CFG: APT-PROXY-PHP-REQUIREMENTS-FILTER-CLASS $cfg->setConfigEntry('apt_proxy_php_requirements_filter_class', 'AptProxyPhpRequirementsFilter'); -// CFG: APT-PROXY-INITIALIZER-FILTER +// CFG: APT-PROXY-INITIALIZER-FILTER-CLASS $cfg->setConfigEntry('apt_proxy_initializer_filter_class', 'AptProxyInitializationFilter'); // CFG: NODE-CONSOLE-APT-PROXY $cfg->setConfigEntry('node_console_apt_proxy_class', 'NodeConsoleAptProxy'); -// CFG: APT-PROXY-WELCOME-TEASER-FILTER +// CFG: APT-PROXY-WELCOME-TEASER-FILTER-CLASS $cfg->setConfigEntry('apt_proxy_welcome_teaser_filter_class', 'AptProxyWelcomeTeaserFilter'); -// CFG: APT-PROXY-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER +// CFG: APT-PROXY-BOOTSTRAP-TASK-HANDLER-INITIALIZER-FILTER-CLASS $cfg->setConfigEntry('apt_proxy_bootstrap_task_handler_initializer_filter_class', 'AptProxyTaskHandlerInitializerFilter'); -// CFG: APT-PROXY-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER +// CFG: APT-PROXY-BOOTSTRAP-EXTRA-BOOTSTRAPPING-FILTER-CLASS $cfg->setConfigEntry('apt_proxy_bootstrap_extra_bootstrapping_filter_class', 'AptProxyBootstrapExtraBootstrappingFilter'); -// CFG: APT-PROXY-BOOTSTRAP-GENERIC-ACTIVATION-FILTER +// CFG: APT-PROXY-BOOTSTRAP-GENERIC-ACTIVATION-FILTER-CLASS $cfg->setConfigEntry('apt_proxy_bootstrap_generic_activation_filter_class', 'AptProxyBootstrapGenericActivationFilter'); -// CFG: APT-PROXY-SHUTDOWN-FILTER +// CFG: APT-PROXY-SHUTDOWN-FILTER-CLASS $cfg->setConfigEntry('apt_proxy_shutdown_filter_class', 'AptProxyShutdownFilter'); // CFG: APT-PROXY-LISTENER-TASK-CLASS diff --git a/core b/core index 93c10cd8b..53bf33122 160000 --- a/core +++ b/core @@ -1 +1 @@ -Subproject commit 93c10cd8bbd88a27bdcd0e65956298666218f150 +Subproject commit 53bf33122fffeedc46ea50c84cbbdd9e2aabd45d -- 2.39.5