Continued with crawler:

[hub.git] / application / hub / main / source / class_BaseUrlSource.php

diff --git a/application/hub/main/source/class_BaseUrlSource.php b/application/hub/main/source/class_BaseUrlSource.php
index afc64492321beb14aac83737695fe5ec7d0c5368..149867f9efbd27add4f1b1a06c2624d2318dcafe 100644 (file)
--- a/application/hub/main/source/class_BaseUrlSource.php
+++ b/application/hub/main/source/class_BaseUrlSource.php
@@ -22,6 +22,9 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 class BaseUrlSource extends BaseSource {
+       // Stack name for all URLs
+       const STACKER_NAME_URLS = 'urls';
+
        // Array elements for CSV data array
        const CRAWL_JOB_ARRAY_START_URL      = 'start_url';
        const CRAWL_JOB_ARRAY_DEPTH          = 'start_depth';
@@ -60,7 +63,7 @@ class BaseUrlSource extends BaseSource {
         */
        public function isUrlStackEmpty () {
                // Determine it
-               $isEmpty = $this->getStackInstance()->isStackEmpty('urls');
+               $isEmpty = $this->getStackInstance()->isStackEmpty(self::STACKER_NAME_URLS);
 
                // Return result
                return $isEmpty;
@@ -75,13 +78,37 @@ class BaseUrlSource extends BaseSource {
         *
         * @param       $crawlData      Array with partial data for being queued
         * @return      void
+        * @todo        ~10% done
         */
        protected function enrichCrawlerQueueData (array &$crawlData) {
                // Debug message
-               /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: crawlData()=' . count($crawlData) . ' - CALLED!');
+               //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: crawlData()=' . count($crawlData) . ' - CALLED!');
+
+               // Check for minimum array elements
+               assert(isset($crawlData[self::CRAWL_JOB_ARRAY_START_URL]));
+               assert(isset($crawlData[self::CRAWL_JOB_ARRAY_DEPTH]));
+
+               // @TODO Add more elements
+
+               // Debug message
+               //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: EXIT!');
+       }
+
+       /**
+        * Enqueues given crawler array in assigned file-based stack
+        *
+        * @param       $crawlData      Array with partial data for being queued
+        * @return      void
+        */
+       protected function enqueueInFileStack (array $crawlData) {
+               // Debug message
+               //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: crawlData()=' . count($crawlData) . ' - CALLED!');
+
+               // Get the stack instance and enqueue it
+               $this->getStackInstance()->pushNamed(self::STACKER_NAME_URLS, $crawlData);
 
                // Debug message
-               /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: EXIT!');
+               //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: EXIT!');
        }
 }