From 413dd541adec999dd857b23b930a425e83acbe2b Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sun, 8 Jul 2018 07:07:09 +0200 Subject: [PATCH] Continued: - no need for scanner class and task, the URL source class can handle this - setApplicationName() is no longer needed - invoke setCommandName() (maybe exists?) instead - Removed crawler instance from BaseHubSystem - moved URL source instance from same class to BaseUrlSourceTask - updated core to latest commit MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Signed-off-by: Roland Häder --- .../hub/classes/class_BaseHubSystem.php | 50 ---------- .../node/class_NodeObjectFactory.php | 3 - .../hub/classes/factories/scanner/.htaccess | 1 - .../scanner/class_ScannerObjectFactory.php | 78 ---------------- .../class_AptProxyInitializationFilter.php | 3 - .../chat/class_ChatInitializationFilter.php | 3 - .../class_CrawlerInitializationFilter.php | 3 - .../class_CruncherInitializationFilter.php | 3 - .../miner/class_MinerInitializationFilter.php | 3 - ...ss_CrawlerTaskHandlerInitializerFilter.php | 10 +- .../class_HubConsoleCommandResolver.php | 6 +- .../html/class_HubHtmlCommandResolver.php | 6 +- .../class_HubConsoleControllerResolver.php | 7 +- .../html/class_HubHtmlControllerResolver.php | 7 +- application/hub/classes/scanner/.htaccess | 1 - application/hub/classes/scanner/class_ | 60 ------------ .../hub/classes/scanner/class_BaseScanner.php | 42 --------- .../hub/classes/scanner/crawler/.htaccess | 1 - .../hub/classes/scanner/crawler/class_Crawler | 60 ------------ .../scanner/crawler/uploaded_list/.htaccess | 1 - .../class_CrawlerUploadedListScanner.php | 70 -------------- .../classes/source/class_BaseUrlSource.php | 16 +++- .../class_CrawlerUploadedListUrlSource.php | 71 +++++++++----- .../tasks/crawler/class_BaseUrlSourceTask.php | 25 ++--- .../classes/tasks/crawler/scanner/.htaccess | 1 - .../tasks/crawler/scanner/class_Crawler | 89 ------------------ .../crawler/scanner/uploaded_list/.htaccess | 1 - .../class_CrawlerUploadedListScannerTask.php | 92 ------------------- .../class_CrawlerUrlSourceFoundRssTask.php | 26 ++++-- .../class_CrawlerUrlSourceLocalStartTask.php | 26 ++++-- .../class_CrawlerUrlSourceRssStartTask.php | 26 ++++-- ...class_CrawlerUrlSourceUploadedListTask.php | 26 ++++-- application/hub/config.php | 15 --- application/hub/interfaces/scanner/.htaccess | 1 - .../hub/interfaces/scanner/class_Scanner.php | 38 -------- core | 2 +- 36 files changed, 155 insertions(+), 718 deletions(-) delete mode 100644 application/hub/classes/factories/scanner/.htaccess delete mode 100644 application/hub/classes/factories/scanner/class_ScannerObjectFactory.php delete mode 100644 application/hub/classes/scanner/.htaccess delete mode 100644 application/hub/classes/scanner/class_ delete mode 100644 application/hub/classes/scanner/class_BaseScanner.php delete mode 100644 application/hub/classes/scanner/crawler/.htaccess delete mode 100644 application/hub/classes/scanner/crawler/class_Crawler delete mode 100644 application/hub/classes/scanner/crawler/uploaded_list/.htaccess delete mode 100644 application/hub/classes/scanner/crawler/uploaded_list/class_CrawlerUploadedListScanner.php delete mode 100644 application/hub/classes/tasks/crawler/scanner/.htaccess delete mode 100644 application/hub/classes/tasks/crawler/scanner/class_Crawler delete mode 100644 application/hub/classes/tasks/crawler/scanner/uploaded_list/.htaccess delete mode 100644 application/hub/classes/tasks/crawler/scanner/uploaded_list/class_CrawlerUploadedListScannerTask.php delete mode 100644 application/hub/interfaces/scanner/.htaccess delete mode 100644 application/hub/interfaces/scanner/class_Scanner.php diff --git a/application/hub/classes/class_BaseHubSystem.php b/application/hub/classes/class_BaseHubSystem.php index db93dde02..1973049ca 100644 --- a/application/hub/classes/class_BaseHubSystem.php +++ b/application/hub/classes/class_BaseHubSystem.php @@ -5,9 +5,7 @@ namespace Org\Shipsimu\Hub\Generic; // Import application-specific stuff use Org\ShipSimu\Hub\Communicator\Communicator; use Org\Shipsimu\Hub\Container\Socket\StorableSocket; -use Org\Shipsimu\Hub\Crawler\Crawler; use Org\Shipsimu\Hub\Crawler\Source\Source; -use Org\Shipsimu\Hub\Crawler\Source\Url\UrlSource; use Org\Shipsimu\Hub\Handler\Protocol\HandleableProtocol; use Org\Shipsimu\Hub\Handler\Network\RawData\HandleableRawData; use Org\Shipsimu\Hub\Information\ShareableInfo; @@ -133,16 +131,6 @@ abstract class BaseHubSystem extends BaseFrameworkSystem implements HubInterface */ private $sourceInstance = NULL; - /** - * An instance of a UrlSource class - */ - private $urlSourceInstance = NULL; - - /** - * An instance of a Crawler class - */ - private $crawlerInstance = NULL; - /** * Name of used protocol */ @@ -507,42 +495,4 @@ abstract class BaseHubSystem extends BaseFrameworkSystem implements HubInterface return $this->sourceInstance; } - /** - * Setter for a UrlSource instance - * - * @param $urlSourceInstance An instance of a UrlSource class - * @return void - */ - public final function setUrlSourceInstance (UrlSource $urlSourceInstance) { - $this->urlSourceInstance = $urlSourceInstance; - } - - /** - * Getter for a UrlSource instance - * - * @return $urlSourceInstance An instance of a UrlSource class - */ - public final function getUrlSourceInstance () { - return $this->urlSourceInstance; - } - - /** - * Setter for a Crawler instance - * - * @param $crawlerInstance An instance of a Crawler class - * @return void - */ - protected final function setCrawlerInstance (Crawler $crawlerInstance) { - $this->crawlerInstance = $crawlerInstance; - } - - /** - * Getter for a Crawler instance - * - * @return $crawlerInstance An instance of a Crawler class - */ - protected final function getCrawlerInstance () { - return $this->crawlerInstance; - } - } diff --git a/application/hub/classes/factories/node/class_NodeObjectFactory.php b/application/hub/classes/factories/node/class_NodeObjectFactory.php index dff30a71b..8ac09fab3 100644 --- a/application/hub/classes/factories/node/class_NodeObjectFactory.php +++ b/application/hub/classes/factories/node/class_NodeObjectFactory.php @@ -83,9 +83,6 @@ class NodeObjectFactory extends ObjectFactory { // Get a registry $applicationInstance = GenericRegistry::getRegistry()->getInstance('application'); - // Set the app instance - $nodeInstance->setApplicationInstance($applicationInstance); - // Add node-specific filters $nodeInstance->addExtraFilters($applicationInstance->getControllerInstance(), $responseInstance); diff --git a/application/hub/classes/factories/scanner/.htaccess b/application/hub/classes/factories/scanner/.htaccess deleted file mode 100644 index 3a4288278..000000000 --- a/application/hub/classes/factories/scanner/.htaccess +++ /dev/null @@ -1 +0,0 @@ -Deny from all diff --git a/application/hub/classes/factories/scanner/class_ScannerObjectFactory.php b/application/hub/classes/factories/scanner/class_ScannerObjectFactory.php deleted file mode 100644 index 94d54b343..000000000 --- a/application/hub/classes/factories/scanner/class_ScannerObjectFactory.php +++ /dev/null @@ -1,78 +0,0 @@ - - * @version 0.0.0 - * @copyright Copyright (c) 2007, 2008 Roland Haeder, 2009 - 2015 Hub Developer Team - * @license GNU GPL 3.0 or any newer version - * @link http://www.shipsimu.org - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -class ScannerObjectFactory extends ObjectFactory { - /** - * Protected constructor - * - * @return void - */ - protected function __construct () { - // Call parent constructor - parent::__construct(__CLASS__); - } - - /** - * Returns a singleton (registry-based) Scanner instance - * - * @param $taskInstance An instance of a class with an Taskable interface - * @return $sourceInstance An instance of a Scanner class - */ - public static final function createScannerInstance (Taskable $taskInstance) { - // Get new factory instance - $factoryInstance = new ScannerObjectFactory(); - - // Get task handler instance - $handlerInstance = GenericRegistry::getRegistry()->getInstance('task_handler'); - - // Get scanner type for given task instance by searching it in task handler - $scannerType = $handlerInstance->searchTask($taskInstance); - - // Only let registered tasks pass - assert(!empty($scannerType)); - - // If there is no handler? - if (GenericRegistry::getRegistry()->instanceExists('scanner_' . $scannerType)) { - // Get handler from registry - $scannerInstance = GenericRegistry::getRegistry()->getInstance('scanner_' . $scannerType); - } else { - // Get the scanner instance - $scannerInstance = ObjectFactory::createObjectByConfiguredName($scannerType); - - // Add it to the registry - GenericRegistry::getRegistry()->addInstance('scanner_' . $scannerType, $scannerInstance); - } - - // Return the instance - return $scannerInstance; - } - -} diff --git a/application/hub/classes/filter/apt-proxy/class_AptProxyInitializationFilter.php b/application/hub/classes/filter/apt-proxy/class_AptProxyInitializationFilter.php index 3b84a675a..8783c83f5 100644 --- a/application/hub/classes/filter/apt-proxy/class_AptProxyInitializationFilter.php +++ b/application/hub/classes/filter/apt-proxy/class_AptProxyInitializationFilter.php @@ -70,9 +70,6 @@ class AptProxyInitializationFilter extends BaseAptProxyFilter implements Filtera // Get a registry $applicationInstance = GenericRegistry::getRegistry()->getInstance('application'); - // Set the app instance - $proxyInstance->setApplicationInstance($applicationInstance); - // Add apt-proxy-specific filters $proxyInstance->addExtraFilters($applicationInstance->getControllerInstance(), $responseInstance); diff --git a/application/hub/classes/filter/chat/class_ChatInitializationFilter.php b/application/hub/classes/filter/chat/class_ChatInitializationFilter.php index b72b70d98..dbb1333f4 100644 --- a/application/hub/classes/filter/chat/class_ChatInitializationFilter.php +++ b/application/hub/classes/filter/chat/class_ChatInitializationFilter.php @@ -70,9 +70,6 @@ class ChatInitializationFilter extends BaseChatFilter implements Filterable { // Get a registry $applicationInstance = GenericRegistry::getRegistry()->getInstance('application'); - // Set the app instance - $chatInstance->setApplicationInstance($applicationInstance); - // Add chat-specific filters $chatInstance->addExtraFilters($applicationInstance->getControllerInstance(), $responseInstance); diff --git a/application/hub/classes/filter/crawler/class_CrawlerInitializationFilter.php b/application/hub/classes/filter/crawler/class_CrawlerInitializationFilter.php index e584121be..a8bede7d3 100644 --- a/application/hub/classes/filter/crawler/class_CrawlerInitializationFilter.php +++ b/application/hub/classes/filter/crawler/class_CrawlerInitializationFilter.php @@ -73,9 +73,6 @@ class CrawlerInitializationFilter extends BaseCrawlerFilter implements Filterabl // Get a registry $applicationInstance = GenericRegistry::getRegistry()->getInstance('application'); - // Set the app instance - $crawlerInstance->setApplicationInstance($applicationInstance); - // Add crawler-specific filters $crawlerInstance->addExtraFilters($applicationInstance->getControllerInstance(), $responseInstance); } diff --git a/application/hub/classes/filter/cruncher/class_CruncherInitializationFilter.php b/application/hub/classes/filter/cruncher/class_CruncherInitializationFilter.php index 0cb44b879..d770106c0 100644 --- a/application/hub/classes/filter/cruncher/class_CruncherInitializationFilter.php +++ b/application/hub/classes/filter/cruncher/class_CruncherInitializationFilter.php @@ -87,9 +87,6 @@ class CruncherInitializationFilter extends BaseCruncherFilter implements Filtera // Get a registry $applicationInstance = GenericRegistry::getRegistry()->getInstance('application'); - // Set the app instance - $cruncherInstance->setApplicationInstance($applicationInstance); - // Add cruncher-specific filters $cruncherInstance->addExtraFilters($applicationInstance->getControllerInstance(), $responseInstance); } catch (ClassNotFoundException $e) { diff --git a/application/hub/classes/filter/miner/class_MinerInitializationFilter.php b/application/hub/classes/filter/miner/class_MinerInitializationFilter.php index 8d468e5ff..e32e8f3a9 100644 --- a/application/hub/classes/filter/miner/class_MinerInitializationFilter.php +++ b/application/hub/classes/filter/miner/class_MinerInitializationFilter.php @@ -87,9 +87,6 @@ class MinerInitializationFilter extends BaseMinerFilter implements Filterable { // Get a registry $applicationInstance = GenericRegistry::getRegistry()->getInstance('application'); - // Set the app instance - $minerInstance->setApplicationInstance($applicationInstance); - // Add miner-specific filters $minerInstance->addExtraFilters($applicationInstance->getControllerInstance(), $responseInstance); } catch (ClassNotFoundException $e) { diff --git a/application/hub/classes/filter/task/crawler/class_CrawlerTaskHandlerInitializerFilter.php b/application/hub/classes/filter/task/crawler/class_CrawlerTaskHandlerInitializerFilter.php index 59b92509c..f34b17710 100644 --- a/application/hub/classes/filter/task/crawler/class_CrawlerTaskHandlerInitializerFilter.php +++ b/application/hub/classes/filter/task/crawler/class_CrawlerTaskHandlerInitializerFilter.php @@ -74,9 +74,6 @@ class CrawlerTaskHandlerInitializerFilter extends BaseCrawlerFilter implements F // Get a new task handler instance $handlerInstance = ObjectFactory::createObjectByConfiguredName('task_handler_class'); - // Put the task handler in registry - GenericRegistry::getRegistry()->addInstance('task_handler', $handlerInstance); - /* * Register all tasks: * @@ -120,15 +117,14 @@ class CrawlerTaskHandlerInitializerFilter extends BaseCrawlerFilter implements F // 10) URL sources foreach (explode(':', $this->getConfigInstance()->getConfigEntry('crawler_url_stacks')) as $stack) { // Init task instance - $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_url_source_' . $stack . '_task_class', array($crawlerInstance)); + $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_url_source_' . $stack . '_task_class'); // And register it $handlerInstance->registerTask('crawler_url_source_' . $stack, $taskInstance); } // END - foreach - // 11) Uploaded list scanner (checks for wanted files) - $taskInstance = ObjectFactory::createObjectByConfiguredName('crawler_uploaded_list_scanner_task_class'); - $handlerInstance->registerTask('crawler_uploaded_list_scanner', $taskInstance); + // Put the task handler in registry + GenericRegistry::getRegistry()->addInstance('task_handler', $handlerInstance); } } diff --git a/application/hub/classes/resolver/command/console/class_HubConsoleCommandResolver.php b/application/hub/classes/resolver/command/console/class_HubConsoleCommandResolver.php index ae0b61c97..c6f5949f3 100644 --- a/application/hub/classes/resolver/command/console/class_HubConsoleCommandResolver.php +++ b/application/hub/classes/resolver/command/console/class_HubConsoleCommandResolver.php @@ -67,12 +67,10 @@ class HubConsoleCommandResolver extends BaseCommandResolver implements CommandRe // Set namespace and application instance $resolverInstance->setNamespace('Org\Shipsimu\Hub\Command'); - $resolverInstance->setApplicationInstance($applicationInstance); + $resolverInstance->setCommandName($commandName); // Return the prepared instance return $resolverInstance; } -} -// [EOF] -?> +} diff --git a/application/hub/classes/resolver/command/html/class_HubHtmlCommandResolver.php b/application/hub/classes/resolver/command/html/class_HubHtmlCommandResolver.php index 9327834fc..3bee470fa 100644 --- a/application/hub/classes/resolver/command/html/class_HubHtmlCommandResolver.php +++ b/application/hub/classes/resolver/command/html/class_HubHtmlCommandResolver.php @@ -67,12 +67,10 @@ class HubHtmlCommandResolver extends BaseCommandResolver implements CommandResol // Set namespace and application instance $resolverInstance->setNamespace('Org\Shipsimu\Hub\Command'); - $resolverInstance->setApplicationInstance($applicationInstance); + $resolverInstance->setCommandName($commandName); // Return the prepared instance return $resolverInstance; } -} -// [EOF] -?> +} diff --git a/application/hub/classes/resolver/controller/console/class_HubConsoleControllerResolver.php b/application/hub/classes/resolver/controller/console/class_HubConsoleControllerResolver.php index 1433bb062..fec6dec70 100644 --- a/application/hub/classes/resolver/controller/console/class_HubConsoleControllerResolver.php +++ b/application/hub/classes/resolver/controller/console/class_HubConsoleControllerResolver.php @@ -66,9 +66,6 @@ class HubConsoleControllerResolver extends BaseControllerResolver implements Con throw new InvalidControllerException(array($resolverInstance, $controllerName), self::EXCEPTION_INVALID_CONTROLLER); } - // Set the application instance - $resolverInstance->setApplicationInstance($applicationInstance); - // Set namespace and command name $resolverInstance->setNamespace('Org\Shipsimu\Hub\Controller'); $resolverInstance->setControllerName($controllerName); @@ -108,7 +105,5 @@ class HubConsoleControllerResolver extends BaseControllerResolver implements Con // Return the maybe resolved instance return $controllerInstance; } -} -// [EOF] -?> +} diff --git a/application/hub/classes/resolver/controller/html/class_HubHtmlControllerResolver.php b/application/hub/classes/resolver/controller/html/class_HubHtmlControllerResolver.php index 2e1299c93..5328f20d4 100644 --- a/application/hub/classes/resolver/controller/html/class_HubHtmlControllerResolver.php +++ b/application/hub/classes/resolver/controller/html/class_HubHtmlControllerResolver.php @@ -66,9 +66,6 @@ class HubHtmlControllerResolver extends BaseControllerResolver implements Contro throw new InvalidControllerException(array($resolverInstance, $controllerName), self::EXCEPTION_INVALID_CONTROLLER); } - // Set the application instance - $resolverInstance->setApplicationInstance($applicationInstance); - // Set namespace and command name $resolverInstance->setNamespace('Org\Shipsimu\Hub\Controller'); $resolverInstance->setControllerName($controllerName); @@ -108,7 +105,5 @@ class HubHtmlControllerResolver extends BaseControllerResolver implements Contro // Return the maybe resolved instance return $controllerInstance; } -} -// [EOF] -?> +} diff --git a/application/hub/classes/scanner/.htaccess b/application/hub/classes/scanner/.htaccess deleted file mode 100644 index 3a4288278..000000000 --- a/application/hub/classes/scanner/.htaccess +++ /dev/null @@ -1 +0,0 @@ -Deny from all diff --git a/application/hub/classes/scanner/class_ b/application/hub/classes/scanner/class_ deleted file mode 100644 index 5cc29c9e5..000000000 --- a/application/hub/classes/scanner/class_ +++ /dev/null @@ -1,60 +0,0 @@ - - * @version 0.0.0 - * @copyright Copyright (c) 2007, 2008 Roland Haeder, 2009 - 2015 Hub Developer Team - * @license GNU GPL 3.0 or any newer version - * @link http://www.ship-simu.org - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -class ???Scanner extends BaseScanner implements Scanner, Registerable { - /** - * Protected constructor - * - * @return void - */ - protected function __construct () { - // Call parent constructor - parent::__construct(__CLASS__); - } - - /** - * Creates an instance of this class - * - * @return $scannerInstance An instance of a Scanner class - */ - public final static function create???Scanner () { - // Get new instance - $scannerInstance = new ???Scanner(); - - // Return the prepared instance - return $scannerInstance; - } - - /** - * Runs the scanner (please no loops here) - * - * @return void - * @todo 0% done - */ - public function execute () { - $this->partialStub('Please implement this method.'); - } -} - -// [EOF] -?> diff --git a/application/hub/classes/scanner/class_BaseScanner.php b/application/hub/classes/scanner/class_BaseScanner.php deleted file mode 100644 index b940f6ed4..000000000 --- a/application/hub/classes/scanner/class_BaseScanner.php +++ /dev/null @@ -1,42 +0,0 @@ - - * @version 0.0.0 - * @copyright Copyright (c) 2007, 2008 Roland Haeder, 2009 - 2015 Hub Developer Team - * @license GNU GPL 3.0 or any newer version - * @link http://www.ship-simu.org - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -abstract class BaseScanner extends BaseHubSystem { - /** - * Protected constructor - * - * @param $className Name of the class - * @return void - */ - protected function __construct ($className) { - // Call parent constructor - parent::__construct($className); - } - -} diff --git a/application/hub/classes/scanner/crawler/.htaccess b/application/hub/classes/scanner/crawler/.htaccess deleted file mode 100644 index 3a4288278..000000000 --- a/application/hub/classes/scanner/crawler/.htaccess +++ /dev/null @@ -1 +0,0 @@ -Deny from all diff --git a/application/hub/classes/scanner/crawler/class_Crawler b/application/hub/classes/scanner/crawler/class_Crawler deleted file mode 100644 index d02f28cd4..000000000 --- a/application/hub/classes/scanner/crawler/class_Crawler +++ /dev/null @@ -1,60 +0,0 @@ - - * @version 0.0.0 - * @copyright Copyright (c) 2014 Crawler Developer Team - * @license GNU GPL 3.0 or any newer version - * @link http://www.ship-simu.org - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -class Crawler???Scanner extends BaseScanner implements Scanner, Registerable { - /** - * Protected constructor - * - * @return void - */ - protected function __construct () { - // Call parent constructor - parent::__construct(__CLASS__); - } - - /** - * Creates an instance of this class - * - * @return $scannerInstance An instance of a Scanner class - */ - public final static function createCrawler???Scanner () { - // Get new instance - $scannerInstance = new Crawler???Scanner(); - - // Return the prepared instance - return $scannerInstance; - } - - /** - * Runs the scanner (please no loops here) - * - * @return void - * @todo 0% done - */ - public function execute () { - $this->partialStub('Please implement this method.'); - } -} - -// [EOF] -?> diff --git a/application/hub/classes/scanner/crawler/uploaded_list/.htaccess b/application/hub/classes/scanner/crawler/uploaded_list/.htaccess deleted file mode 100644 index 3a4288278..000000000 --- a/application/hub/classes/scanner/crawler/uploaded_list/.htaccess +++ /dev/null @@ -1 +0,0 @@ -Deny from all diff --git a/application/hub/classes/scanner/crawler/uploaded_list/class_CrawlerUploadedListScanner.php b/application/hub/classes/scanner/crawler/uploaded_list/class_CrawlerUploadedListScanner.php deleted file mode 100644 index 0a09b066e..000000000 --- a/application/hub/classes/scanner/crawler/uploaded_list/class_CrawlerUploadedListScanner.php +++ /dev/null @@ -1,70 +0,0 @@ - - * @version 0.0.0 - * @copyright Copyright (c) 2014 Crawler Developer Team - * @license GNU GPL 3.0 or any newer version - * @link http://www.ship-simu.org - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -class CrawlerUploadedListScanner extends BaseScanner implements Scanner, Registerable { - /** - * Protected constructor - * - * @return void - */ - protected function __construct () { - // Call parent constructor - parent::__construct(__CLASS__); - } - - /** - * Creates an instance of this class - * - * @return $scannerInstance An instance of a Scanner class - */ - public final static function createCrawlerUploadedListScanner () { - // Get new instance - $scannerInstance = new CrawlerUploadedListScanner(); - - // Return the prepared instance - return $scannerInstance; - } - - /** - * Runs the scanner (please no loops here) - * - * @return void - * @todo 0% done - */ - public function execute () { - $this->partialStub('Please implement this method.'); - } -} - -// [EOF] -?> diff --git a/application/hub/classes/source/class_BaseUrlSource.php b/application/hub/classes/source/class_BaseUrlSource.php index 26ae79894..e8f59c048 100644 --- a/application/hub/classes/source/class_BaseUrlSource.php +++ b/application/hub/classes/source/class_BaseUrlSource.php @@ -58,11 +58,17 @@ abstract class BaseUrlSource extends BaseSource { * @return void */ protected function initSource ($prefix, $sourceName) { + // Trace message + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: prefix=' . $prefix . ',sourceName=' . $sourceName . ' - CALLED!'); + // Use another object factory $stackInstance = FileStackFactory::createFileStackInstance($prefix . '_url', $sourceName); // Set the stack here $this->setStackInstance($stackInstance); + + // Trace message + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: EXIT!'); } /** @@ -87,11 +93,11 @@ abstract class BaseUrlSource extends BaseSource { * * @param $crawlData Array with partial data for being queued * @return void - * @todo ~10% done + * @todo ~1% done */ protected function enrichCrawlerQueueData (array &$crawlData) { // Debug message - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : crawlData()=' . count($crawlData) . ' - CALLED!'); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: crawlData()=' . count($crawlData) . ' - CALLED!'); // Check for minimum array elements assert(isset($crawlData[self::CRAWL_JOB_ARRAY_START_URL])); @@ -100,7 +106,7 @@ abstract class BaseUrlSource extends BaseSource { // @TODO Add more elements // Debug message - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : EXIT!'); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: EXIT!'); } /** @@ -111,13 +117,13 @@ abstract class BaseUrlSource extends BaseSource { */ protected function enqueueInFileStack (array $crawlData) { // Debug message - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : crawlData()=' . count($crawlData) . ' - CALLED!'); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: crawlData()=' . count($crawlData) . ' - CALLED!'); // Get the stack instance and enqueue it $this->getStackInstance()->pushNamed(self::STACKER_NAME_URLS, $crawlData); // Debug message - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : EXIT!'); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: EXIT!'); } } diff --git a/application/hub/classes/source/urls/class_CrawlerUploadedListUrlSource.php b/application/hub/classes/source/urls/class_CrawlerUploadedListUrlSource.php index 214cd2209..763dec022 100644 --- a/application/hub/classes/source/urls/class_CrawlerUploadedListUrlSource.php +++ b/application/hub/classes/source/urls/class_CrawlerUploadedListUrlSource.php @@ -11,6 +11,9 @@ use Org\Mxchange\CoreFramework\Factory\ObjectFactory; use Org\Mxchange\CoreFramework\Generic\NullPointerException; use Org\Mxchange\CoreFramework\Registry\Registerable; +// Import SPL stuff +use \SplFileInfo; + /** * A UploadedList URL source class for crawlers * @@ -88,7 +91,10 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R parent::__construct(__CLASS__); // "Cache" CSV path for faster usage - $this->csvFilePath = $this->getConfigInstance()->getConfigEntry('root_base_path') . '/' . $this->getConfigInstance()->getConfigEntry('crawler_csv_file_path'); + $this->csvFilePath = $this->getConfigInstance()->getConfigEntry('root_base_path') . $this->getConfigInstance()->getConfigEntry('crawler_csv_file_path'); + + // Debug message + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: this->csvFilePath=' . $this->csvFilePath); // Initialize directory instance $directoryInstance = ObjectFactory::createObjectByConfiguredName('directory_class', array($this->csvFilePath)); @@ -105,6 +111,9 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R // "Cache" column separator $this->columnSeparator = $this->getConfigInstance()->getConfigEntry('crawler_url_list_column_separator'); + + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: Finished contructing object.'); } /** @@ -113,33 +122,40 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R * @return $isFound Whether a CSV file is found */ private function isCsvFileFound () { - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : CALLED!'); + // Trace message + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: CALLED!'); // Is it valid? if (!$this->getDirectoryInstance()->getDirectoryIteratorInstance()->valid()) { + // Debug message + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: Rewinding iterator ...'); + // Rewind to start $this->getDirectoryInstance()->getDirectoryIteratorInstance()->rewind(); } // END - if // Read next entry - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : this->csvFileImported=' . print_r($this->csvFileImported, TRUE)); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: this->csvFileImported=' . print_r($this->csvFileImported, TRUE)); $directoryEntry = $this->getDirectoryInstance()->readDirectoryExcept(array_merge(array('.htaccess', '.', '..'), $this->csvFileImported)); // Debug message - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE directoryEntry(' . strlen($directoryEntry) . ')=' . $directoryEntry); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: directoryEntry(' . strlen($directoryEntry) . ')=' . $directoryEntry); // Is it empty or wrong file extension? if ((empty($directoryEntry)) || (substr($directoryEntry, -4, 4) != '.csv')) { // Skip further processing - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE directoryEntry(' . strlen($directoryEntry) . ')=' . $directoryEntry . ' - SKIPPED!'); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: directoryEntry(' . strlen($directoryEntry) . ')=' . $directoryEntry . ' - SKIPPED!'); return FALSE; } // END - if + // Init SPL file instance + $infoInstance = new SplFileInfo($this->csvFilePath . '/' . $directoryEntry); + // Initialize CSV file instance - $this->lastCsvFileInstance = ObjectFactory::createObjectByConfiguredName('csv_input_file_class', array($this->csvFilePath . '/' . $directoryEntry)); + $this->lastCsvFileInstance = ObjectFactory::createObjectByConfiguredName('csv_input_file_class', array($infoInstance)); // Debug message - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE directoryEntry(' . strlen($directoryEntry) . ')=' . $directoryEntry . ' - Instance created - EXIT!'); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: directoryEntry(' . strlen($directoryEntry) . ')=' . $directoryEntry . ' - Instance created - EXIT!'); // Found an entry return TRUE; @@ -151,6 +167,9 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R * @return $sourceInstance An instance of an UrlSource class */ public final static function createCrawlerUploadedListUrlSource () { + // Debug message + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: - CONSTRUCTOR!'); + // Get new instance $sourceInstance = new CrawlerUploadedListUrlSource(); @@ -171,7 +190,7 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R */ private function saveCsvDataInCrawlerQueue (array $csvData) { // Debug message - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : csvData()=' . count($csvData) . ' - CALLED!'); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: csvData()=' . count($csvData) . ' - CALLED!'); // The array must have a fixed amount of elements, later enhancements may accept more assert(count($csvData) == self::CRAWL_ENTRY_SIZE); @@ -188,13 +207,13 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R ); // Debug message - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : csvArray()=' . count($csvArray) . ' - BEFORE!'); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: csvArray()=' . count($csvArray) . ' - BEFORE!'); // Then add more data to it $this->enrichCrawlerQueueData($csvArray); // Debug message - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : csvArray()=' . count($csvArray) . ' - AFTER!'); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: csvArray()=' . count($csvArray) . ' - AFTER!'); /* * Then enqueue it in the file stack. The local crawler "task" will @@ -203,7 +222,7 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R $this->enqueueInFileStack($csvArray); // Debug message - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : EXIT!'); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: EXIT!'); } /** @@ -239,24 +258,32 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R * @throws NullPointerException If lastCsvFileInstance is not set */ private function addCsvFile () { - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : CALLED!'); + // Trace message + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: CALLED!'); // Is the instance set? if (is_null($this->lastCsvFileInstance)) { // This should not happen throw new NullPointerException($this, self::EXCEPTION_IS_NULL_POINTER); - } // END - if + } elseif (in_array($this->lastCsvFileInstance->getFileObject()->getBasename(), $this->csvFileImported)) { + // Already stacked + return; + } + + // Debug message + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: this->lastCsvFileInstance=' . $this->lastCsvFileInstance); // Stack this file $this->getStackSourceInstance()->pushNamed(self::STACK_NAME_CSV_FILE, $this->lastCsvFileInstance); // ... and mark it as "imported" - array_push($this->csvFileImported, basename($this->lastCsvFileInstance->getFileName())); + array_push($this->csvFileImported, $this->lastCsvFileInstance->getFileObject()->getBasename()); // ... and finally NULL it (to save some RAM) $this->lastCsvFileInstance = NULL; - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : EXIT!'); + // Trace message + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: EXIT!'); } /** @@ -268,7 +295,7 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R */ private function parseCsvFile () { // Debug message - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : CALLED!'); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: CALLED!'); // Get next entry $csvFileInstance = $this->getStackSourceInstance()->popNamed(self::STACK_NAME_CSV_FILE); @@ -277,7 +304,7 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R $csvData = $csvFileInstance->readCsvFileLine($this->columnSeparator); // Debug message - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : csvData[' . gettype($csvData) . ']=' . print_r($csvData, TRUE)); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: csvData[' . gettype($csvData) . ']=' . print_r($csvData, TRUE)); // Expect always an array assert(is_array($csvData)); @@ -285,7 +312,7 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R // Is the array empty? if (count($csvData) == 0) { // Debug message - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : File ' . $csvFileInstance->getFileName() . ' has been fully read.'); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: File ' . $csvFileInstance->getFilename() . ' has been fully read.'); // Try to close it by actually unsetting (destructing) it unset($csvFileInstance); @@ -307,7 +334,7 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R $this->getStackSourceInstance()->pushNamed(self::STACK_NAME_CSV_ENTRY, $csvData); // Debug message - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : EXIT!'); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: EXIT!'); } /** @@ -317,13 +344,13 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R */ private function parseCsvEntry () { // Debug message - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : CALLED!'); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: CALLED!'); // Pop it from stack $csvData = $this->getStackSourceInstance()->popNamed(self::STACK_NAME_CSV_ENTRY); // Debug message - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : csvData[' . gettype($csvData) . ']=' . print_r($csvData, TRUE)); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: csvData[' . gettype($csvData) . ']=' . print_r($csvData, TRUE)); // It must have a fixed amount of elements (see method parseCsvFile() for details) assert(count($csvData) == self::CRAWL_ENTRY_SIZE); @@ -332,7 +359,7 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R $this->saveCsvDataInCrawlerQueue($csvData); // Debug message - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE : EXIT!'); + /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-SOURCE: EXIT!'); } /** diff --git a/application/hub/classes/tasks/crawler/class_BaseUrlSourceTask.php b/application/hub/classes/tasks/crawler/class_BaseUrlSourceTask.php index a92d4c955..f5a9dd18d 100644 --- a/application/hub/classes/tasks/crawler/class_BaseUrlSourceTask.php +++ b/application/hub/classes/tasks/crawler/class_BaseUrlSourceTask.php @@ -3,7 +3,7 @@ namespace Org\Shipsimu\Hub\Crawler\Task\Source\Url; // Import application-specific stuff -use Org\Shipsimu\Hub\Crawler\Crawler; +use Org\Shipsimu\Hub\Crawler\Source\Url\UrlSource; use Org\Shipsimu\Hub\Crawler\Source\Url\UrlSourceObjectFactory; // Import framework stuff @@ -32,10 +32,11 @@ use Org\Mxchange\CoreFramework\Task\BaseTask; * along with this program. If not, see . */ abstract class BaseUrlSourceTask extends BaseTask { + /** - * An instance of a Crawler class + * An instance of a UrlSource class */ - private $crawlerInstance = NULL; + private $urlSourceInstance = NULL; /** * Protected constructor @@ -58,26 +59,26 @@ abstract class BaseUrlSourceTask extends BaseTask { $sourceInstance = UrlSourceObjectFactory::createUrlSourceInstance($this); // And set it here - $this->getCrawlerInstance()->setUrlSourceInstance($sourceInstance); + $this->setUrlSourceInstance($sourceInstance); } /** - * Setter for a Crawler instance + * Setter for a UrlSource instance * - * @param $crawlerInstance An instance of a Crawler class + * @param $urlSourceInstance An instance of a UrlSource class * @return void */ - protected final function setCrawlerInstance (Crawler $crawlerInstance) { - $this->crawlerInstance = $crawlerInstance; + private function setUrlSourceInstance (UrlSource $urlSourceInstance) { + $this->urlSourceInstance = $urlSourceInstance; } /** - * Getter for a Crawler instance + * Getter for a UrlSource instance * - * @return $crawlerInstance An instance of a Crawler class + * @return $urlSourceInstance An instance of a UrlSource class */ - protected final function getCrawlerInstance () { - return $this->crawlerInstance; + protected final function getUrlSourceInstance () { + return $this->urlSourceInstance; } } diff --git a/application/hub/classes/tasks/crawler/scanner/.htaccess b/application/hub/classes/tasks/crawler/scanner/.htaccess deleted file mode 100644 index 3a4288278..000000000 --- a/application/hub/classes/tasks/crawler/scanner/.htaccess +++ /dev/null @@ -1 +0,0 @@ -Deny from all diff --git a/application/hub/classes/tasks/crawler/scanner/class_Crawler b/application/hub/classes/tasks/crawler/scanner/class_Crawler deleted file mode 100644 index 9f0258632..000000000 --- a/application/hub/classes/tasks/crawler/scanner/class_Crawler +++ /dev/null @@ -1,89 +0,0 @@ - - * @version 0.0.0 - * @copyright Copyright (c) 2014 Crawler Developer Team - * @license GNU GPL 3.0 or any newer version - * @link http://www.ship-simu.org - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -class Crawler???ScannerTask extends BaseTask implements Taskable, Visitable { - /** - * Protected constructor - * - * @return void - */ - protected function __construct () { - // Call parent constructor - parent::__construct(__CLASS__); - } - - /** - * Creates an instance of this class - * - * @return $taskInstance An instance of a Visitable class - */ - public final static function createCrawler???ScannerTask () { - // Get new instance - $taskInstance = new Crawler???ScannerTask(); - - // Return the prepared instance - return $taskInstance; - } - - /** - * Accepts the visitor to process the visitor - * - * @param $visitorInstance An instance of a Visitor class - * @return void - * @todo Maybe visit some sub-objects - */ - public function accept (Visitor $visitorInstance) { - // Visit this task - $visitorInstance->visitTask($this); - } - - /** - * Executes the task - * - * @return void - */ - public function executeTask () { - // Call factory for a scanner instance and execute it - ScannerFactory::createScannerInstance($this)->execute(); - } - - /** - * Shuts down the task - * - * @return void - * @todo 0% done - */ - public function doShutdown () { - self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-TASK: Shutting down...'); - } - -} diff --git a/application/hub/classes/tasks/crawler/scanner/uploaded_list/.htaccess b/application/hub/classes/tasks/crawler/scanner/uploaded_list/.htaccess deleted file mode 100644 index 3a4288278..000000000 --- a/application/hub/classes/tasks/crawler/scanner/uploaded_list/.htaccess +++ /dev/null @@ -1 +0,0 @@ -Deny from all diff --git a/application/hub/classes/tasks/crawler/scanner/uploaded_list/class_CrawlerUploadedListScannerTask.php b/application/hub/classes/tasks/crawler/scanner/uploaded_list/class_CrawlerUploadedListScannerTask.php deleted file mode 100644 index 0249172c6..000000000 --- a/application/hub/classes/tasks/crawler/scanner/uploaded_list/class_CrawlerUploadedListScannerTask.php +++ /dev/null @@ -1,92 +0,0 @@ - - * @version 0.0.0 - * @copyright Copyright (c) 2014 Crawler Developer Team - * @license GNU GPL 3.0 or any newer version - * @link http://www.ship-simu.org - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -class CrawlerUploadedListScannerTask extends BaseTask implements Taskable, Visitable { - /** - * Protected constructor - * - * @return void - */ - protected function __construct () { - // Call parent constructor - parent::__construct(__CLASS__); - } - - /** - * Creates an instance of this class - * - * @return $taskInstance An instance of a Visitable class - */ - public final static function createCrawlerUploadedListScannerTask () { - // Get new instance - $taskInstance = new CrawlerUploadedListScannerTask(); - - // Return the prepared instance - return $taskInstance; - } - - /** - * Accepts the visitor to process the visitor - * - * @param $visitorInstance An instance of a Visitor class - * @return void - * @todo Maybe visit some sub-objects - */ - public function accept (Visitor $visitorInstance) { - // Visit this task - $visitorInstance->visitTask($this); - } - - /** - * Executes the task - * - * @return void - */ - public function executeTask () { - // Call factory for a scanner instance and execute it - ScannerObjectFactory::createScannerInstance($this)->execute(); - } - - /** - * Shuts down the task - * - * @return void - * @todo 0% done - */ - public function doShutdown () { - self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-UPLOADED-LIST-SCANNER-TASK: Shutting down...'); - } - -} diff --git a/application/hub/classes/tasks/crawler/url_source/class_CrawlerUrlSourceFoundRssTask.php b/application/hub/classes/tasks/crawler/url_source/class_CrawlerUrlSourceFoundRssTask.php index f0c61174a..cbc7ed910 100644 --- a/application/hub/classes/tasks/crawler/url_source/class_CrawlerUrlSourceFoundRssTask.php +++ b/application/hub/classes/tasks/crawler/url_source/class_CrawlerUrlSourceFoundRssTask.php @@ -3,7 +3,6 @@ namespace Org\Shipsimu\Hub\Crawler\Task\Source\FoundRss; // Import application-specific stuff -use Org\Shipsimu\Hub\Crawler\Crawler; use Org\Shipsimu\Hub\Crawler\Task\Source\Url\BaseUrlSourceTask; // Import framework stuff @@ -47,16 +46,12 @@ class CrawlerUrlSourceFoundRssTask extends BaseUrlSourceTask implements Taskable /** * Creates an instance of this class * - * @param $crawlerInstance An instance of a Crawler class * @return $taskInstance An instance of a Visitable class */ - public final static function createCrawlerUrlSourceFoundRssTask (Crawler $crawlerInstance) { + public final static function createCrawlerUrlSourceFoundRssTask () { // Get new instance $taskInstance = new CrawlerUrlSourceFoundRssTask(); - // Set crawler instance here - $taskInstance->setCrawlerInstance($crawlerInstance); - // Return the prepared instance return $taskInstance; } @@ -79,20 +74,35 @@ class CrawlerUrlSourceFoundRssTask extends BaseUrlSourceTask implements Taskable * @return void */ public function executeTask () { + // Trace message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: CALLED!'); + // Get source instance - $sourceInstance = $this->getCrawlerInstance()->getUrlSourceInstance(); + $sourceInstance = $this->getUrlSourceInstance(); + + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: sourceInstance[]=' . gettype($sourceInstance)); // Is it not set? if (is_null($sourceInstance)) { + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: Initializing source ...'); + // Initialize it $this->initUrlSourceTask(); // And re-get it - $sourceInstance = $this->getCrawlerInstance()->getUrlSourceInstance(); + $sourceInstance = $this->getUrlSourceInstance(); } // END - if + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: sourceInstance[]=' . gettype($sourceInstance)); + // Get the URL source instance and fill the stack with crawl entries $sourceInstance->fillUrlStack(); + + // Trace message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: EXIT!'); } /** diff --git a/application/hub/classes/tasks/crawler/url_source/class_CrawlerUrlSourceLocalStartTask.php b/application/hub/classes/tasks/crawler/url_source/class_CrawlerUrlSourceLocalStartTask.php index 7eabde2df..b8a0f21d6 100644 --- a/application/hub/classes/tasks/crawler/url_source/class_CrawlerUrlSourceLocalStartTask.php +++ b/application/hub/classes/tasks/crawler/url_source/class_CrawlerUrlSourceLocalStartTask.php @@ -3,7 +3,6 @@ namespace Org\Shipsimu\Hub\Crawler\Task\Source\LocalStart; // Import application-specific stuff -use Org\Shipsimu\Hub\Crawler\Crawler; use Org\Shipsimu\Hub\Crawler\Task\Source\Url\BaseUrlSourceTask; // Import framework stuff @@ -47,16 +46,12 @@ class CrawlerUrlSourceLocalStartTask extends BaseUrlSourceTask implements Taskab /** * Creates an instance of this class * - * @param $crawlerInstance An instance of a Crawler class * @return $taskInstance An instance of a Visitable class */ - public final static function createCrawlerUrlSourceLocalStartTask (Crawler $crawlerInstance) { + public final static function createCrawlerUrlSourceLocalStartTask () { // Get new instance $taskInstance = new CrawlerUrlSourceLocalStartTask(); - // Set crawler instance here - $taskInstance->setCrawlerInstance($crawlerInstance); - // Return the prepared instance return $taskInstance; } @@ -79,20 +74,35 @@ class CrawlerUrlSourceLocalStartTask extends BaseUrlSourceTask implements Taskab * @return void */ public function executeTask () { + // Trace message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: CALLED!'); + // Get source instance - $sourceInstance = $this->getCrawlerInstance()->getUrlSourceInstance(); + $sourceInstance = $this->getUrlSourceInstance(); + + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: sourceInstance[]=' . gettype($sourceInstance)); // Is it not set? if (is_null($sourceInstance)) { + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: Initializing source ...'); + // Initialize it $this->initUrlSourceTask(); // And re-get it - $sourceInstance = $this->getCrawlerInstance()->getUrlSourceInstance(); + $sourceInstance = $this->getUrlSourceInstance(); } // END - if + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: sourceInstance[]=' . gettype($sourceInstance)); + // Get the URL source instance and fill the stack with crawl entries $sourceInstance->fillUrlStack(); + + // Trace message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: EXIT!'); } /** diff --git a/application/hub/classes/tasks/crawler/url_source/class_CrawlerUrlSourceRssStartTask.php b/application/hub/classes/tasks/crawler/url_source/class_CrawlerUrlSourceRssStartTask.php index 0f07ba11f..5a67eb3b5 100644 --- a/application/hub/classes/tasks/crawler/url_source/class_CrawlerUrlSourceRssStartTask.php +++ b/application/hub/classes/tasks/crawler/url_source/class_CrawlerUrlSourceRssStartTask.php @@ -3,7 +3,6 @@ namespace Org\Shipsimu\Hub\Crawler\Task\Source\RssStart; // Import application-specific stuff -use Org\Shipsimu\Hub\Crawler\Crawler; use Org\Shipsimu\Hub\Crawler\Task\Source\Url\BaseUrlSourceTask; // Import framework stuff @@ -47,16 +46,12 @@ class CrawlerUrlSourceRssStartTask extends BaseUrlSourceTask implements Taskable /** * Creates an instance of this class * - * @param $crawlerInstance An instance of a Crawler class * @return $taskInstance An instance of a Visitable class */ - public final static function createCrawlerUrlSourceRssStartTask (Crawler $crawlerInstance) { + public final static function createCrawlerUrlSourceRssStartTask () { // Get new instance $taskInstance = new CrawlerUrlSourceRssStartTask(); - // Set crawler instance here - $taskInstance->setCrawlerInstance($crawlerInstance); - // Return the prepared instance return $taskInstance; } @@ -79,20 +74,35 @@ class CrawlerUrlSourceRssStartTask extends BaseUrlSourceTask implements Taskable * @return void */ public function executeTask () { + // Trace message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: CALLED!'); + // Get source instance - $sourceInstance = $this->getCrawlerInstance()->getUrlSourceInstance(); + $sourceInstance = $this->getUrlSourceInstance(); + + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: sourceInstance[]=' . gettype($sourceInstance)); // Is it not set? if (is_null($sourceInstance)) { + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: Initializing source ...'); + // Initialize it $this->initUrlSourceTask(); // And re-get it - $sourceInstance = $this->getCrawlerInstance()->getUrlSourceInstance(); + $sourceInstance = $this->getUrlSourceInstance(); } // END - if + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: sourceInstance[]=' . gettype($sourceInstance)); + // Get the URL source instance and fill the stack with crawl entries $sourceInstance->fillUrlStack(); + + // Trace message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: EXIT!'); } /** diff --git a/application/hub/classes/tasks/crawler/url_source/class_CrawlerUrlSourceUploadedListTask.php b/application/hub/classes/tasks/crawler/url_source/class_CrawlerUrlSourceUploadedListTask.php index 15a6960c4..908c6f6ba 100644 --- a/application/hub/classes/tasks/crawler/url_source/class_CrawlerUrlSourceUploadedListTask.php +++ b/application/hub/classes/tasks/crawler/url_source/class_CrawlerUrlSourceUploadedListTask.php @@ -3,7 +3,6 @@ namespace Org\Shipsimu\Hub\Crawler\Task\Source\UploadedList; // Import application-specific stuff -use Org\Shipsimu\Hub\Crawler\Crawler; use Org\Shipsimu\Hub\Crawler\Task\Source\Url\BaseUrlSourceTask; // Import framework stuff @@ -47,16 +46,12 @@ class CrawlerUrlSourceUploadedListTask extends BaseUrlSourceTask implements Task /** * Creates an instance of this class * - * @param $crawlerInstance An instance of a Crawler class * @return $taskInstance An instance of a Visitable class */ - public final static function createCrawlerUrlSourceUploadedListTask (Crawler $crawlerInstance) { + public final static function createCrawlerUrlSourceUploadedListTask () { // Get new instance $taskInstance = new CrawlerUrlSourceUploadedListTask(); - // Set cralwer instance here - $taskInstance->setCrawlerInstance($crawlerInstance); - // Return the prepared instance return $taskInstance; } @@ -79,20 +74,35 @@ class CrawlerUrlSourceUploadedListTask extends BaseUrlSourceTask implements Task * @return void */ public function executeTask () { + // Trace message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: CALLED!'); + // Get source instance - $sourceInstance = $this->getCrawlerInstance()->getUrlSourceInstance(); + $sourceInstance = $this->getUrlSourceInstance(); + + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: sourceInstance[]=' . gettype($sourceInstance)); // Is it not set? if (is_null($sourceInstance)) { + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: Initializing source ...'); + // Initialize it $this->initUrlSourceTask(); // And re-get it - $sourceInstance = $this->getCrawlerInstance()->getUrlSourceInstance(); + $sourceInstance = $this->getUrlSourceInstance(); } // END - if + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: sourceInstance[]=' . gettype($sourceInstance)); + // Get the URL source instance and fill the stack with crawl entries $sourceInstance->fillUrlStack(); + + // Trace message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('CRAWLER-URL-SOURCE-UPLOADED-LIST-TASK: EXIT!'); } /** diff --git a/application/hub/config.php b/application/hub/config.php index 0509ddfff..f427c15c4 100644 --- a/application/hub/config.php +++ b/application/hub/config.php @@ -1295,9 +1295,6 @@ $cfg->setConfigEntry('crawler_booting_state_class', 'Org\Shipsimu\Hub\State\Craw // CFG: COMMUNICATOR-INIT-STATE-CLASS $cfg->setConfigEntry('communicator_init_state_class', 'Org\Shipsimu\Hub\State\Communicator\CommunicatorInitState'); -// CFG: CRAWLER-UPLOADED-LIST-SCANNER -$cfg->setConfigEntry('crawler_uploaded_list_scanner', 'Org\Shipsimu\Hub\Crawler\Scanner\UploadedList\CrawlerUploadedListScanner'); - // CFG: CRAWLER-URL-SOURCE-LOCAL-START-CLASS $cfg->setConfigEntry('crawler_url_source_local_start_class', 'Org\Shipsimu\Hub\Crawler\Source\LocalStart\CrawlerLocalStartUrlSource'); @@ -1502,18 +1499,6 @@ $cfg->setConfigEntry('task_crawler_url_source_found_rss_interval_delay', 150); // CFG: TASK-CRAWLER-URL-SOURCE-FOUND-RSS-MAX-RUNS $cfg->setConfigEntry('task_crawler_url_source_found_rss_max_runs', 0); -// CFG: CRAWLER-URL-SOURCE-FOUND-RSS-TASK-CLASS -$cfg->setConfigEntry('crawler_uploaded_list_scanner_task_class', 'Org\Shipsimu\Hub\Crawler\Task\Scanner\UploadedList\CrawlerUploadedListScannerTask'); - -// CFG: TASK-CRAWLER-URL-SOURCE-FOUND-RSS-STARTUP-DELAY -$cfg->setConfigEntry('task_crawler_uploaded_list_scanner_startup_delay', 500); - -// CFG: TASK-CRAWLER-URL-SOURCE-FOUND-RSS-INTERVAL-DELAY -$cfg->setConfigEntry('task_crawler_uploaded_list_scanner_interval_delay', 1000); - -// CFG: TASK-CRAWLER-URL-SOURCE-FOUND-RSS-MAX-RUNS -$cfg->setConfigEntry('task_crawler_uploaded_list_scanner_max_runs', 0); - // CFG: CRAWLER-CSV-FILE-PATH $cfg->setConfigEntry('crawler_csv_file_path', 'data/url_lists'); diff --git a/application/hub/interfaces/scanner/.htaccess b/application/hub/interfaces/scanner/.htaccess deleted file mode 100644 index 3a4288278..000000000 --- a/application/hub/interfaces/scanner/.htaccess +++ /dev/null @@ -1 +0,0 @@ -Deny from all diff --git a/application/hub/interfaces/scanner/class_Scanner.php b/application/hub/interfaces/scanner/class_Scanner.php deleted file mode 100644 index 65183e719..000000000 --- a/application/hub/interfaces/scanner/class_Scanner.php +++ /dev/null @@ -1,38 +0,0 @@ - - * @version 0.0.0 - * @copyright Copyright (c) 2007, 2008 Roland Haeder, 2009 - 2015 Hub Developer Team - * @license GNU GPL 3.0 or any newer version - * @link http://www.shipsimu.org - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -interface Scanner extends HubInterface { - /** - * Runs the scanner (please no loops here) - * - * @return void - */ - function execute (); - -} diff --git a/core b/core index 3191b3ee4..868c87760 160000 --- a/core +++ b/core @@ -1 +1 @@ -Subproject commit 3191b3ee467d8844ec574321291cfc6ddc34eb69 +Subproject commit 868c877607670760eb36e63ebeb1a04237907be9 -- 2.39.5