application/hub/main/source/urls/class_CrawlerUploadedListUrlSource.php

   1 <?php
   2 /**
   3  * A UploadedList URL source class for crawlers
   4  *
   5  * @author              Roland Haeder <webmaster@ship-simu.org>
   6  * @version             0.0.0
   7  * @copyright   Copyright (c) 2014 Crawler Developer Team
   8  * @license             GNU GPL 3.0 or any newer version
   9  * @link                http://www.ship-simu.org
  10  *
  11  * This program is free software: you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License as published by
  13  * the Free Software Foundation, either version 3 of the License, or
  14  * (at your option) any later version.
  15  *
  16  * This program is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  * GNU General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU General Public License
  22  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  23  */
  24 class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, Registerable {
  25         /**
  26          * "Cached" CSV path
  27          */
  28         private $csvFilePath = '';
  29
  30         /**
  31          * Last CSV file instance
  32          */
  33         private $lastCsvFileInstance = NULL;
  34
  35         /**
  36          * Stack for pushing data from this clas to another
  37          */
  38         private $stackSourceInstance = NULL;
  39
  40         /**
  41          * Stack name for a CSV file
  42          */
  43         const STACK_NAME_CSV_FILE = 'csv_file';
  44
  45         /**
  46          * "Imported" CSV files
  47          */
  48         private $csvFileImported = array();
  49
  50         /**
  51          * Protected constructor
  52          *
  53          * @return      void
  54          */
  55         protected function __construct () {
  56                 // Call parent constructor
  57                 parent::__construct(__CLASS__);
  58
  59                 // "Cache" CSV path for faster usage
  60                 $this->csvFilePath = $this->getConfigInstance()->getConfigEntry('base_path') . '/' . $this->getConfigInstance()->getConfigEntry('crawler_csv_file_path');
  61
  62                 // Initialize directory instance
  63                 $directoryInstance = ObjectFactory::createObjectByConfiguredName('directory_class', array($this->csvFilePath));
  64
  65                 // Set it here
  66                 $this->setDirectoryInstance($directoryInstance);
  67
  68                 // Init stack instance
  69                 $this->stackSourceInstance = ObjectFactory::createObjectByConfiguredName('crawler_uploaded_list_url_source_stack_class');
  70
  71                 // Init stack
  72                 $this->getStackSourceInstance()->initStack(self::STACK_NAME_CSV_FILE);
  73         }
  74
  75         /**
  76          * Checks whether a CSV file is found in configured path
  77          *
  78          * @return      $isFound        Whether a CSV file is found
  79          */
  80         private function isCsvFileFound () {
  81                 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: CALLED!');
  82
  83                 // Is it valid?
  84                 if (!$this->getDirectoryInstance()->getDirectoryIteratorInstance()->valid()) {
  85                         // Rewind to start
  86                         $this->getDirectoryInstance()->getDirectoryIteratorInstance()->rewind();
  87                 } // END - if
  88
  89                 // Read next entry
  90                 /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: this->csvFileImported=' . print_r($this->csvFileImported, TRUE));
  91                 $directoryEntry = $this->getDirectoryInstance()->readDirectoryExcept(array_merge(array('.htaccess', '.', '..'), $this->csvFileImported));
  92
  93                 // Debug message
  94                 /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE[' . __METHOD__ . ':' . __LINE__ . '] directoryEntry(' . strlen($directoryEntry) . ')=' . $directoryEntry);
  95
  96                 // Is it empty or wrong file extension?
  97                 if ((empty($directoryEntry)) || (substr($directoryEntry, -4, 4) != '.csv')) {
  98                         // Skip further processing
  99                         /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE[' . __METHOD__ . ':' . __LINE__ . '] directoryEntry(' . strlen($directoryEntry) . ')=' . $directoryEntry . ' - SKIPPED!');
 100                         return FALSE;
 101                 } // END - if
 102
 103                 // Initialize CSV file instance
 104                 $this->lastCsvFileInstance = ObjectFactory::createObjectByConfiguredName('csv_file_class', array($this->csvFilePath . '/' . $directoryEntry));
 105
 106                 // Found an entry
 107                 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: EXIT!');
 108                 return TRUE;
 109         }
 110
 111         /**
 112          * Creates an instance of this class
 113          *
 114          * @return      $sourceInstance         An instance of a Source class
 115          */
 116         public final static function createCrawlerUploadedListUrlSource () {
 117                 // Get new instance
 118                 $sourceInstance = new CrawlerUploadedListUrlSource();
 119
 120                 // Init source
 121                 $sourceInstance->initSource('crawler', 'uploaded_list');
 122
 123                 // Return the prepared instance
 124                 return $sourceInstance;
 125         }
 126
 127         /**
 128          * Initializes the import of the CSV file which is being processed by other task
 129          *
 130          * @return      void
 131          * @throws      NullPointerException    If lastCsvFileInstance is not set
 132          */
 133         private function importCsvFile () {
 134                 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: CALLED!');
 135
 136                 // Is the instance set?
 137                 if (is_null($this->lastCsvFileInstance)) {
 138                         // This should not happen
 139                         throw new NullPointerException($this, self::EXCEPTION_IS_NULL_POINTER);
 140                 } // END - if
 141
 142                 // Stack this file
 143                 $this->getStackSourceInstance()->pushNamed(self::STACK_NAME_CSV_FILE, $this->lastCsvFileInstance);
 144
 145                 // ... and mark it as "imported"
 146                 array_push($this->csvFileImported, basename($this->lastCsvFileInstance->getFileName()));
 147
 148                 // ... and finally NULL it (to save some RAM)
 149                 $this->lastCsvFileInstance = NULL;
 150
 151                 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: EXIT!');
 152         }
 153
 154         /**
 155          * Getter for stackSourceInstance variable
 156          *
 157          * @return      $stackSourceInstance    An instance of an additional stack
 158          */
 159         public final function getStackSourceInstance () {
 160                 return $this->stackSourceInstance;
 161         }
 162
 163         /**
 164          * Processes entries in the stack.
 165          *
 166          * @return      void
 167          * @todo        ~10% done
 168          */
 169         public function processStack () {
 170                 // Does the stack have some entries left?
 171                 if (!$this->isUrlStackEmpty()) {
 172                         /*
 173                          * Handle next entry. This method will be called very often, so need
 174                          * to process more than one entry at a time.
 175                          */
 176                         $this->processNextEntry();
 177                 } elseif ($this->isCsvFileFound()) {
 178                         /*
 179                          * A file containing an URL list is found. Please note the format is
 180                          * CSV-like as you may wish to provide meta data such as crawl
 181                          * depth, handling of 3rd-party URLs and such.
 182                          */
 183                         $this->importCsvFile();
 184                 }
 185
 186                 $this->partialStub('Please implement this method.');
 187         }
 188 }
 189
 190 // [EOF]
 191 ?>