+++ /dev/null
-<?php
-/**
- * A UploadedList URL source class for crawlers
- *
- * @author Roland Haeder <webmaster@ship-simu.org>
- * @version 0.0.0
- * @copyright Copyright (c) 2014 Crawler Developer Team
- * @license GNU GPL 3.0 or any newer version
- * @link http://www.ship-simu.org
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, Registerable {
- /**
- * Stack name for a CSV file
- */
- const STACK_NAME_CSV_FILE = 'csv_file';
-
- /**
- * Stack name for a CSV entry
- */
- const STACK_NAME_CSV_ENTRY = 'csv_entry';
-
- /**
- * Size of crawl (CSV) entry which is an indexed array:
- *
- * 0 = URL to crawl
- * 1 = Crawl depth of URL
- * 2 = Crawl depth of linked URLs (same other host only)
- */
- const CRAWL_ENTRY_SIZE = 3;
-
- /**
- * "Cached" CSV path
- */
- private $csvFilePath = '';
-
- /**
- * Last CSV file instance
- */
- private $lastCsvFileInstance = NULL;
-
- /**
- * Stack for pushing data from this clas to another
- */
- private $stackSourceInstance = NULL;
-
- /**
- * "Imported" CSV files
- */
- private $csvFileImported = array();
-
- /**
- * "Cached" separator for columns
- */
- private $columnSeparator = '';
-
- /**
- * Protected constructor
- *
- * @return void
- */
- protected function __construct () {
- // Call parent constructor
- parent::__construct(__CLASS__);
-
- // "Cache" CSV path for faster usage
- $this->csvFilePath = $this->getConfigInstance()->getConfigEntry('base_path') . '/' . $this->getConfigInstance()->getConfigEntry('crawler_csv_file_path');
-
- // Initialize directory instance
- $directoryInstance = ObjectFactory::createObjectByConfiguredName('directory_class', array($this->csvFilePath));
-
- // Set it here
- $this->setDirectoryInstance($directoryInstance);
-
- // Init stack instance
- $this->stackSourceInstance = ObjectFactory::createObjectByConfiguredName('crawler_uploaded_list_url_source_stack_class');
-
- // Init stacks
- $this->getStackSourceInstance()->initStack(self::STACK_NAME_CSV_FILE);
- $this->getStackSourceInstance()->initStack(self::STACK_NAME_CSV_ENTRY);
-
- // "Cache" column separator
- $this->columnSeparator = $this->getConfigInstance()->getConfigEntry('crawler_url_list_column_separator');
- }
-
- /**
- * Checks whether a CSV file is found in configured path
- *
- * @return $isFound Whether a CSV file is found
- */
- private function isCsvFileFound () {
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: CALLED!');
-
- // Is it valid?
- if (!$this->getDirectoryInstance()->getDirectoryIteratorInstance()->valid()) {
- // Rewind to start
- $this->getDirectoryInstance()->getDirectoryIteratorInstance()->rewind();
- } // END - if
-
- // Read next entry
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: this->csvFileImported=' . print_r($this->csvFileImported, TRUE));
- $directoryEntry = $this->getDirectoryInstance()->readDirectoryExcept(array_merge(array('.htaccess', '.', '..'), $this->csvFileImported));
-
- // Debug message
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE[' . __METHOD__ . ':' . __LINE__ . '] directoryEntry(' . strlen($directoryEntry) . ')=' . $directoryEntry);
-
- // Is it empty or wrong file extension?
- if ((empty($directoryEntry)) || (substr($directoryEntry, -4, 4) != '.csv')) {
- // Skip further processing
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE[' . __METHOD__ . ':' . __LINE__ . '] directoryEntry(' . strlen($directoryEntry) . ')=' . $directoryEntry . ' - SKIPPED!');
- return FALSE;
- } // END - if
-
- // Initialize CSV file instance
- $this->lastCsvFileInstance = ObjectFactory::createObjectByConfiguredName('csv_input_file_class', array($this->csvFilePath . '/' . $directoryEntry));
-
- // Debug message
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . '] directoryEntry(' . strlen($directoryEntry) . ')=' . $directoryEntry . ' - Instance created - EXIT!');
-
- // Found an entry
- return TRUE;
- }
-
- /**
- * Creates an instance of this class
- *
- * @return $sourceInstance An instance of a Source class
- */
- public final static function createCrawlerUploadedListUrlSource () {
- // Get new instance
- $sourceInstance = new CrawlerUploadedListUrlSource();
-
- // Init source
- $sourceInstance->initSource('crawler', 'uploaded_list');
-
- // Return the prepared instance
- return $sourceInstance;
- }
-
- /**
- * Enriches and saves the given CSV entry (array) in the assigned
- * file-based stack. To such entry a lot more informations are added, such
- * as which files shall be crawled and many more.
- *
- * @param $csvData Array with data from a CSV file
- * @return void
- */
- private function saveCsvDataInCrawlerQueue (array $csvData) {
- // Debug message
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: csvData()=' . count($csvData) . ' - CALLED!');
-
- // The array must have a fixed amount of elements, later enhancements may accept more
- assert(count($csvData) == self::CRAWL_ENTRY_SIZE);
-
- /*
- * First converted the indexed array into an assoziative array. Don't
- * forget to expand this array as well when you want to add another
- * column to the CSV file.
- */
- $csvArray = array(
- self::CRAWL_JOB_ARRAY_START_URL => $csvData[0],
- self::CRAWL_JOB_ARRAY_DEPTH => $csvData[1],
- self::CRAWL_JOB_ARRAY_EXTERNAL_DEPTH => $csvData[2]
- );
-
- // Debug message
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: csvArray()=' . count($csvArray) . ' - BEFORE!');
-
- // Then add more data to it
- $this->enrichCrawlerQueueData($csvArray);
-
- // Debug message
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: csvArray()=' . count($csvArray) . ' - AFTER!');
-
- /*
- * Then enqueue it in the file stack. The local crawler "task" will
- * then pick this up.
- */
- $this->enqueueInFileStack($csvArray);
-
- // Debug message
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: EXIT!');
- }
-
- /**
- * Checks whether a CSV file has been loaded (added to the stack)
- *
- * @return $isAdded Whether a CSV file has been loaded
- */
- private function isCsvFileAdded () {
- // Check whether the stacker is not empty
- $isAdded = (($this->getStackSourceInstance()->isStackInitialized(self::STACK_NAME_CSV_FILE)) && (!$this->getStackSourceInstance()->isStackEmpty(self::STACK_NAME_CSV_FILE)));
-
- // Return the result
- return $isAdded;
- }
-
- /**
- * Checks whether a CSV entry has been added to the stack
- *
- * @return $isAdded Whether a CSV entry has been added
- */
- private function isCsvEntryAdded () {
- // Check whether the stacker is not empty
- $isAdded = (($this->getStackSourceInstance()->isStackInitialized(self::STACK_NAME_CSV_ENTRY)) && (!$this->getStackSourceInstance()->isStackEmpty(self::STACK_NAME_CSV_ENTRY)));
-
- // Return the result
- return $isAdded;
- }
-
- /**
- * Initializes the import of the CSV file which is being processed by other task
- *
- * @return void
- * @throws NullPointerException If lastCsvFileInstance is not set
- */
- private function addCsvFile () {
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: CALLED!');
-
- // Is the instance set?
- if (is_null($this->lastCsvFileInstance)) {
- // This should not happen
- throw new NullPointerException($this, self::EXCEPTION_IS_NULL_POINTER);
- } // END - if
-
- // Stack this file
- $this->getStackSourceInstance()->pushNamed(self::STACK_NAME_CSV_FILE, $this->lastCsvFileInstance);
-
- // ... and mark it as "imported"
- array_push($this->csvFileImported, basename($this->lastCsvFileInstance->getFileName()));
-
- // ... and finally NULL it (to save some RAM)
- $this->lastCsvFileInstance = NULL;
-
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: EXIT!');
- }
-
- /**
- * Parses the next stacked CSV file by reading only one line from it. Then
- * the read line is being validated and if found good being feed to the next
- * stack. The file is removed from stack only if it has been fully parsed.
- *
- * @return void
- */
- private function parseCsvFile () {
- // Debug message
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: CALLED!');
-
- // Get next entry
- $csvFileInstance = $this->getStackSourceInstance()->popNamed(self::STACK_NAME_CSV_FILE);
-
- // Read full "CSV line"
- $csvData = $csvFileInstance->readCsvFileLine($this->columnSeparator);
-
- // Debug message
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: csvData[' . gettype($csvData) . ']=' . print_r($csvData, TRUE));
-
- // Expect always an array
- assert(is_array($csvData));
-
- // Is the array empty?
- if (count($csvData) == 0) {
- // Debug message
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: File ' . $csvFileInstance->getFileName() . ' has been fully read.');
-
- // Try to close it by actually unsetting (destructing) it
- unset($csvFileInstance);
-
- // This file as been fully read, so don't push it back on stack.
- return;
- } // END - if
-
- // ... with a fixed amount of elements, later enhancements may accept more
- assert(count($csvData) == self::CRAWL_ENTRY_SIZE);
-
- /*
- * Push the file back on stack as it may contain more entries. This way
- * all files got rotated on stack which may improve crawler performance.
- */
- $this->getStackSourceInstance()->pushNamed(self::STACK_NAME_CSV_FILE, $csvFileInstance);
-
- // Push array on next stack
- $this->getStackSourceInstance()->pushNamed(self::STACK_NAME_CSV_ENTRY, $csvData);
-
- // Debug message
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: EXIT!');
- }
-
- /**
- * Parses the next stacked CSV entry.
- *
- * @return void
- */
- private function parseCsvEntry () {
- // Debug message
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: CALLED!');
-
- // Pop it from stack
- $csvData = $this->getStackSourceInstance()->popNamed(self::STACK_NAME_CSV_ENTRY);
-
- // Debug message
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: csvData[' . gettype($csvData) . ']=' . print_r($csvData, TRUE));
-
- // It must have a fixed amount of elements (see method parseCsvFile() for details)
- assert(count($csvData) == self::CRAWL_ENTRY_SIZE);
-
- // Save it in crawler queue (which will enrich it with way more informations
- $this->saveCsvDataInCrawlerQueue($csvData);
-
- // Debug message
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: EXIT!');
- }
-
- /**
- * Getter for stackSourceInstance variable
- *
- * @return $stackSourceInstance An instance of an additional stack
- */
- public final function getStackSourceInstance () {
- return $this->stackSourceInstance;
- }
-
- /**
- * Fills the URL stack with new entries from source
- *
- * @return void
- * @todo ~40% done
- */
- public function fillUrlStack () {
- // Does the stack have some entries left?
- if ($this->isCsvEntryAdded()) {
- /*
- * A CSV file has been found and "imported" (added to stack). Now
- * the file can be read line by line and checked every one of it.
- */
- $this->parseCsvEntry();
- } elseif ($this->isCsvFileAdded()) {
- /*
- * A CSV file has been found and "imported" (added to stack). Now
- * the file can be read line by line and checked every one of it.
- */
- $this->parseCsvFile();
- } elseif ($this->isCsvFileFound()) {
- /*
- * A file containing an URL list is found. Please note the format is
- * CSV-like as you may wish to provide meta data such as crawl
- * depth, handling of 3rd-party URLs and such.
- */
- $this->addCsvFile();
- }
-
- $this->partialStub('Please implement this method.');
- }
-}
-
-// [EOF]
-?>
projects / hub.git / blobdiff