Fixes for crawler:

[hub.git] / application / hub / main / source / urls / class_CrawlerUploadedListUrlSource.php

diff --git a/application/hub/main/source/urls/class_CrawlerUploadedListUrlSource.php b/application/hub/main/source/urls/class_CrawlerUploadedListUrlSource.php
index 749c3f631ed66821cb3da2b55c6aff0f6e930851..33caffaf89b054a5b7f48db6f60628c798af5835 100644 (file)
--- a/application/hub/main/source/urls/class_CrawlerUploadedListUrlSource.php
+++ b/application/hub/main/source/urls/class_CrawlerUploadedListUrlSource.php
@@ -159,9 +159,9 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R
         */
        private function saveCsvDataInCrawlerQueue (array $csvData) {
                // Debug message
-               /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: csvData()=' . count($csvData) . ' - CALLED!');
+               //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: csvData()=' . count($csvData) . ' - CALLED!');
 
-               // The array has 3 elements, later enhancements may accept more
+               // The array must have a fixed amount of elements, later enhancements may accept more
                assert(count($csvData) == self::CRAWL_ENTRY_SIZE);
 
                /*
@@ -175,11 +175,23 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R
                        self::CRAWL_JOB_ARRAY_EXTERNAL_DEPTH => $csvData[2]
                );
 
+               // Debug message
+               //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: csvArray()=' . count($csvArray) . ' - BEFORE!');
+
                // Then add more data to it
-               $this->enrichCrawlerQueueData($csvData);
+               $this->enrichCrawlerQueueData($csvArray);
 
                // Debug message
-               /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: EXIT!');
+               //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: csvArray()=' . count($csvArray) . ' - AFTER!');
+
+               /*
+                * Then enqueue it in the file stack. The local crawler "task" will
+                * then pick this up.
+                */
+               $this->enqueueInFileStack($csvArray);
+
+               // Debug message
+               //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: EXIT!');
        }
 
        /**
@@ -263,14 +275,14 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R
                        // Debug message
                        //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: File ' . $csvFileInstance->getFileName() . ' has been fully read.');
 
-                       // Try to close it
-                       $csvFileInstance->closeFile();
+                       // Try to close it by actually unsetting (destructing) it
+                       unset($csvFileInstance);
 
                        // This file as been fully read, so don't push it back on stack.
                        return;
                } // END - if
 
-               // ...  with 3 elements, later enhancements may accept more
+               // ...  with a fixed amount of elements, later enhancements may accept more
                assert(count($csvData) == self::CRAWL_ENTRY_SIZE);
 
                /*
@@ -293,22 +305,22 @@ class CrawlerUploadedListUrlSource extends BaseUrlSource implements UrlSource, R
         */
        private function parseCsvEntry () {
                // Debug message
-               /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: CALLED!');
+               //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: CALLED!');
 
                // Pop it from stack
                $csvData = $this->getStackSourceInstance()->popNamed(self::STACK_NAME_CSV_ENTRY);
 
                // Debug message
-               /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: csvData[' . gettype($csvData) . ']=' . print_r($csvData, TRUE));
+               //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: csvData[' . gettype($csvData) . ']=' . print_r($csvData, TRUE));
 
-               // It must have 3 elements (see method parseCsvFile() for details)
+               // It must have a fixed amount of elements (see method parseCsvFile() for details)
                assert(count($csvData) == self::CRAWL_ENTRY_SIZE);
 
                // Save it in crawler queue (which will enrich it with way more informations
                $this->saveCsvDataInCrawlerQueue($csvData);
 
                // Debug message
-               /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: EXIT!');
+               //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('CRAWLER-SOURCE [' . __METHOD__ . ':' . __LINE__ . ']: EXIT!');
        }
 
        /**