X-Git-Url: https://git.mxchange.org/?p=core.git;a=blobdiff_plain;f=inc%2Fclasses%2Fmain%2Fstacker%2Ffile%2Fclass_BaseFileStack.php;h=4aa880bb9aae1e33c8d1fd86015d011c4e544f9f;hp=b7597f0c9c4393c2f522be42078517c3dda90797;hb=c367a220d41fd9d44a88da15777433035b55b560;hpb=f9cf8e7eb7bc57c332c61e35fbbbc6fae356313b diff --git a/inc/classes/main/stacker/file/class_BaseFileStack.php b/inc/classes/main/stacker/file/class_BaseFileStack.php index b7597f0c..4aa880bb 100644 --- a/inc/classes/main/stacker/file/class_BaseFileStack.php +++ b/inc/classes/main/stacker/file/class_BaseFileStack.php @@ -28,44 +28,24 @@ class BaseFileStack extends BaseStacker { const STACK_MAGIC = 'STACKv0.1'; /** - * Separator magic->count + * Separator for header data */ - const SEPARATOR_MAGIC_COUNT = 0x00; + const SEPARATOR_HEADER_DATA = 0x01; /** - * Separator position->entries + * Separator header->entries */ - const SEPARATOR_SEEK_POS_ENTRIES = 0xff; + const SEPARATOR_HEADER_ENTRIES = 0x02; /** * Separator hash->name */ - const SEPARATOR_HASH_NAME = 0x05; + const SEPARATOR_HASH_NAME = 0x03; /** * Length of name */ - const COUNT_NAME = 10; - - /** - * Length of count - */ - const COUNT_LENGTH = 20; - - /** - * Length of position - */ - const COUNT_POSITION = 20; - - /** - * Counter for total entries - */ - private $totalEntries = 0; - - /** - * Current seek position - */ - private $seekPosition = 0; + const LENGTH_NAME = 10; /** * Protected constructor @@ -76,138 +56,80 @@ class BaseFileStack extends BaseStacker { protected function __construct ($className) { // Call parent constructor parent::__construct($className); - } - - /** - * Getter for total entries - * - * @return $totalEntries Total entries in this stack - */ - private function getCounter () { - // Get it - return $this->totalEntries; - } - /** - * Increment counter - * - * @return void - */ - private function incrementCounter () { - // Get it - $this->totalEntries++; - } - - /** - * Getter for seek position - * - * @return $seekPosition Current seek position (stored here in object) - */ - private function getSeekPosition () { - // Get it - return $this->seekPosition; - } + // Calculate header size + $this->setHeaderSize( + strlen(self::STACK_MAGIC) + + strlen(self::SEPARATOR_HEADER_DATA) + + self::LENGTH_COUNT + + strlen(self::SEPARATOR_HEADER_DATA) + + self::LENGTH_POSITION + + strlen(self::SEPARATOR_HEADER_ENTRIES) + ); - /** - * Setter for seek position - * - * @param $seekPosition Current seek position (stored here in object) - * @return void - */ - private function setSeekPosition ($seekPosition) { - // And set it - $this->seekPosition = $seekPosition; + // Init counters and gaps array + $this->initCountersGapsArray(); } /** - * Updates seekPosition attribute from file to avoid to much access on file. + * Reads the file header * * @return void */ - private function updateSeekPosition () { - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__)); + protected function readFileHeader () { + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__)); - // Get key (= seek position) - $seekPosition = $this->getIteratorInstance()->key(); - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Setting seekPosition=%s', __METHOD__, __LINE__, $seekPosition)); + // First rewind to beginning as the header sits at the beginning ... + $this->getIteratorInstance()->rewind(); - // And set it here - $this->setSeekPosition($seekPosition); + // Then read it (see constructor for calculation) + $data = $this->getIteratorInstance()->read($this->getHeaderSize()); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Read %d bytes (%d wanted).', __METHOD__, __LINE__, strlen($data), $this->getHeaderSize())); - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__)); - } + // Have all requested bytes been read? + assert(strlen($data) == $this->getHeaderSize()); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__)); - /** - * Checks whether the file header is initialized - * - * @return $isInitialized Whether the file header is initialized - */ - private function isFileHeaderInitialized () { - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__)); - // Default is not initialized - $isInitialized = FALSE; - - // Is the file initialized? - if ($this->isFileInitialized()) { - // Some bytes has been written, so rewind to start of it. - $rewindStatus = $this->getIteratorInstance()->rewind(); - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] rewindStatus=%s', __METHOD__, __LINE__, $rewindStatus)); - - // Is the rewind() call successfull? - if ($rewindStatus != 1) { - // Something bad happened - self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Could not rewind().', __METHOD__, __LINE__)); - } // END - if - - // Read file header - $this->readFileHeader(); - } // END - if + // Last character must be the separator + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] data(-1)=%s', __METHOD__, __LINE__, dechex(ord(substr($data, -1, 1))))); + assert(substr($data, -1, 1) == chr(self::SEPARATOR_HEADER_ENTRIES)); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__)); - // Return result - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] isInitialized=%d - EXIT!', __METHOD__, __LINE__, intval($isInitialized))); - return $isInitialized; - } + // Okay, then remove it + $data = substr($data, 0, -1); - /** - * Checks whether the file-based stack has been initialized - * - * @return $isInitialized Whether the file's size is zero - */ - private function isFileInitialized () { - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__)); - - // Get it from iterator which holds the pointer instance. If FALSE is returned - $fileSize = $this->getIteratorInstance()->size(); - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] fileSize=%s', __METHOD__, __LINE__, $fileSize)); + // And update seek position + $this->updateSeekPosition(); /* - * The returned file size should not be FALSE or NULL as this means - * that the pointer class does not work correctly. + * Now split it: + * + * 0 => magic + * 1 => total entries + * 2 => current seek position */ - assert(is_int($fileSize)); + $this->header = explode(chr(self::SEPARATOR_HEADER_DATA), $data); - // Is more than 0 returned? - $isInitialized = ($fileSize > 0); + // Check if the array has only 3 elements + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] header(%d)=%s', __METHOD__, __LINE__, count($this->header), print_r($this->header, TRUE))); + assert(count($this->header) == 3); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__)); - // Return result - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] isInitialized=%d - EXIT!', __METHOD__, __LINE__, intval($isInitialized))); - return $isInitialized; - } + // Check magic + assert($this->header[0] == self::STACK_MAGIC); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__)); - /** - * Creates the file-stack's header - * - * @return void - */ - private function createFileHeader () { - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__)); - // The file's header should not be initialized here - assert(!$this->isFileHeaderInitialized()); + // Check length of count and seek position + assert(strlen($this->header[1]) == self::LENGTH_COUNT); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__)); + assert(strlen($this->header[2]) == self::LENGTH_POSITION); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__)); - // Flush file header - $this->flushFileHeader(); + // Decode count and seek position + $this->header[1] = hex2bin($this->header[1]); + $this->header[2] = hex2bin($this->header[2]); - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!!', __METHOD__, __LINE__)); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__)); } /** @@ -216,80 +138,66 @@ class BaseFileStack extends BaseStacker { * @return void */ private function flushFileHeader () { - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__)); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__)); // Put all informations together - $header = sprintf('%s%s%s%s%s', + $header = sprintf('%s%s%s%s%s%s', // Magic self::STACK_MAGIC, // Separator magic<->count - chr(self::SEPARATOR_MAGIC_COUNT), + chr(self::SEPARATOR_HEADER_DATA), // Total entries (will be zero) and pad it to 20 chars - str_pad($this->dec2hex($this->getCounter()), self::COUNT_LENGTH, '0', STR_PAD_LEFT), + str_pad($this->dec2hex($this->getCounter()), self::LENGTH_COUNT, '0', STR_PAD_LEFT), + + // Separator count<->seek position + chr(self::SEPARATOR_HEADER_DATA), // Position (will be zero) - str_pad($this->dec2hex(0, 2), self::COUNT_POSITION, '0', STR_PAD_LEFT), + str_pad($this->dec2hex($this->getSeekPosition(), 2), self::LENGTH_POSITION, '0', STR_PAD_LEFT), // Separator position<->entries - chr(self::SEPARATOR_SEEK_POS_ENTRIES) + chr(self::SEPARATOR_HEADER_ENTRIES) ); // Write it to disk (header is always at seek position 0) - $this->getIteratorInstance()->writeAtPosition(0, $header); + $this->writeData(0, $header); - // Update seek position - $this->updateSeekPosition(); - - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__)); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__)); } /** - * Pre-allocates file (if enabled) with some space for later faster write access. + * Analyzes entries in stack file. This will count all found (and valid) + * entries, mark invalid as damaged and count gaps ("fragmentation"). If + * only gaps are found, the file is considered as "virgin" (no entries). * * @return void */ - private function preAllocateFile () { - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__)); - - // Is it enabled? - if ($this->getConfigInstance()->getConfigEntry('file_stack_pre_allocate_enabled') != 'Y') { - // Not enabled - self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Not pre-allocating stack file.', __METHOD__, __LINE__)); + private function analyzeFile () { + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__)); - // Don't continue here. - return; - } // END - if - - // Message to user - self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Pre-allocating stack file ...', __METHOD__, __LINE__)); - - /* - * Calculate minimum length for one entry: - * minimum length = hash length + separator + name + minimum entry size = ?? + 1 + 10 + 1 = ?? - */ - $minLengthEntry = self::getHashLength() + strlen(self::SEPARATOR_HASH_NAME) + self::COUNT_NAME + 1; - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] minLengthEntry=%s', __METHOD__, __LINE__, $minLengthEntry)); + // Make sure the file is initialized + assert($this->isFileInitialized()); - // Calulcate seek position - $seekPosition = $minLengthEntry * $this->getConfigInstanstance()->getConfigEntry('file_stack_pre_allocate_count'); - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] seekPosition=%s', __METHOD__, __LINE__, $seekPosition)); + // Init counters and gaps array + $this->initCountersGapsArray(); - // Now seek to the position - $this->getIteratorInstance()->seek($seekPosition); + // Output message (as this may take some time) + self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Analyzing file structure ... (this may take some time)', __METHOD__, __LINE__)); - /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__)); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__)); } /** * Initializes this file-based stack. * * @param $fileName File name of this stack + * @param $type Type of this stack (e.g. url_source for URL sources) * @return void */ - protected function initFileStack ($fileName) { - // Get a file i/o pointer instance + protected function initFileStack ($fileName, $type) { + // Get a file i/o pointer instance for stack file $pointerInstance = ObjectFactory::createObjectByConfiguredName('file_raw_input_output_class', array($fileName)); // Get iterator instance @@ -302,72 +210,28 @@ class BaseFileStack extends BaseStacker { $this->setIteratorInstance($iteratorInstance); // Is the file's header initialized? - if ($this->isFileHeaderInitialized()) { - // Then load it - $this->loadFileHeader(); - } else { + if (!$this->isFileHeaderInitialized()) { // No, then create it (which may pre-allocate the stack) $this->createFileHeader(); // And pre-allocate a bit - $this->preAllocateFile(); - } - } - - /** - * Initializes given stacker - * - * @param $stackerName Name of the stack - * @param $forceReInit Force re-initialization - * @return void - * @throws AlreadyInitializedStackerException If the stack is already initialized - */ - public function initStack ($stackerName, $forceReInit = FALSE) { - // Is the stack already initialized? - if (($forceReInit === FALSE) && ($this->isStackInitialized($stackerName))) { - // Then throw the exception - throw new AlreadyInitializedStackerException(array($this, $stackerName, $forceReInit), self::EXCEPTION_STACKER_ALREADY_INITIALIZED); + $this->preAllocateFile('file_stack'); } // END - if - // Initialize the given stack - $this->partialStub('stackerName=' . $stackerName . ',forceReInit=' . intval($forceReInit)); - } - - /** - * Checks whether the given stack is initialized (set in array $stackers) - * - * @param $stackerName Name of the stack - * @return $isInitialized Whether the stack is initialized - */ - public function isStackInitialized ($stackerName) { - // Is is there? - $this->partialStub('stackerName=' . $stackerName); - $isInitialized = TRUE; - - // Return result - return $isInitialized; - } + // Load the file header + $this->readFileHeader(); - /** - * Getter for size of given stack (array count) - * - * @param $stackerName Name of the stack - * @return $count Size of stack (array count) - * @throws NoStackerException If given stack is missing - */ - public function getStackCount ($stackerName) { - // Is the stack not yet initialized? - if (!$this->isStackInitialized($stackerName)) { - // Throw an exception - throw new NoStackerException(array($this, $stackerName), self::EXCEPTION_NO_STACKER_FOUND); - } // END - if + // Count all entries in file + $this->analyzeFile(); - // Now, count the array of entries - $this->partialStub('stackerName=' . $stackerName); - $count = 0; + /* + * Get stack index instance. This can be used for faster + * "defragmentation" and startup. + */ + $indexInstance = FileStackIndexFactory::createFileStackIndexInstance($fileName, $type); - // Return result - return $count; + // And set it here + $this->setIndexInstance($indexInstance); } /** @@ -376,17 +240,14 @@ class BaseFileStack extends BaseStacker { * @param $stackerName Name of the stack * @param $value Value to add to this stacker * @return void - * @throws FullStackerException Thrown if the stack is full + * @throws FullStackerException If the stack is full */ protected function addValue ($stackerName, $value) { - // Is the stack not yet initialized or full? - if (!$this->isStackInitialized($stackerName)) { - // Then do it here - $this->initStack($stackerName); - } elseif ($this->isStackFull($stackerName)) { + // Do some tests + if ($this->isStackFull($stackerName)) { // Stacker is full throw new FullStackerException(array($this, $stackerName, $value), self::EXCEPTION_STACKER_IS_FULL); - } + } // END - if // Now add the value to the stack $this->partialStub('stackerName=' . $stackerName . ',value[]=' . gettype($value)); @@ -397,18 +258,14 @@ class BaseFileStack extends BaseStacker { * * @param $stackerName Name of the stack * @return $value Value of last added value - * @throws NoStackerException If the named stacker was not found - * @throws EmptyStackerException If the named stacker is empty + * @throws EmptyStackerException If the stack is empty */ protected function getLastValue ($stackerName) { // Is the stack not yet initialized or full? - if (!$this->isStackInitialized($stackerName)) { - // Throw an exception - throw new NoStackerException(array($this, $stackerName), self::EXCEPTION_NO_STACKER_FOUND); - } elseif ($this->isStackEmpty($stackerName)) { + if ($this->isStackEmpty($stackerName)) { // Throw an exception throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY); - } + } // END - if // Now get the last value $this->partialStub('stackerName=' . $stackerName); @@ -423,18 +280,14 @@ class BaseFileStack extends BaseStacker { * * @param $stackerName Name of the stack * @return $value Value of last added value - * @throws NoStackerException If the named stacker was not found - * @throws EmptyStackerException If the named stacker is empty + * @throws EmptyStackerException If the stack is empty */ protected function getFirstValue ($stackerName) { // Is the stack not yet initialized or full? - if (!$this->isStackInitialized($stackerName)) { - // Throw an exception - throw new NoStackerException(array($this, $stackerName), self::EXCEPTION_NO_STACKER_FOUND); - } elseif ($this->isStackEmpty($stackerName)) { + if ($this->isStackEmpty($stackerName)) { // Throw an exception throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY); - } + } // END - if // Now get the first value $this->partialStub('stackerName=' . $stackerName); @@ -449,18 +302,14 @@ class BaseFileStack extends BaseStacker { * * @param $stackerName Name of the stack * @return $value Value "poped" from array - * @throws NoStackerException If the named stacker was not found - * @throws EmptyStackerException If the named stacker is empty + * @throws EmptyStackerException If the stack is empty */ protected function popLast ($stackerName) { // Is the stack not yet initialized or full? - if (!$this->isStackInitialized($stackerName)) { - // Throw an exception - throw new NoStackerException(array($this, $stackerName), self::EXCEPTION_NO_STACKER_FOUND); - } elseif ($this->isStackEmpty($stackerName)) { + if ($this->isStackEmpty($stackerName)) { // Throw an exception throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY); - } + } // END - if // Now, remove the last entry, we don't care about the return value here, see elseif() block above $this->partialStub('stackerName=' . $stackerName); @@ -472,23 +321,93 @@ class BaseFileStack extends BaseStacker { * * @param $stackerName Name of the stack * @return $value Value "shifted" from array - * @throws NoStackerException If the named stacker was not found * @throws EmptyStackerException If the named stacker is empty */ protected function popFirst ($stackerName) { // Is the stack not yet initialized or full? - if (!$this->isStackInitialized($stackerName)) { - // Throw an exception - throw new NoStackerException(array($this, $stackerName), self::EXCEPTION_NO_STACKER_FOUND); - } elseif ($this->isStackEmpty($stackerName)) { + if ($this->isStackEmpty($stackerName)) { // Throw an exception throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY); - } + } // END - if // Now, remove the last entry, we don't care about the return value here, see elseif() block above $this->partialStub('stackerName=' . $stackerName); return NULL; } + + /** + * Checks whether the given stack is full + * + * @param $stackerName Name of the stack + * @return $isFull Whether the stack is full + */ + protected function isStackFull ($stackerName) { + // File-based stacks will only run full if the disk space is low. + // @TODO Please implement this, returning FALSE + $isFull = FALSE; + + // Return result + return $isFull; + } + + /** + * Checks whether the given stack is empty + * + * @param $stackerName Name of the stack + * @return $isEmpty Whether the stack is empty + * @throws NoStackerException If given stack is missing + */ + public function isStackEmpty ($stackerName) { + // So, is the stack empty? + $isEmpty = (($this->getStackCount($stackerName)) == 0); + + // Return result + return $isEmpty; + } + + /** + * Initializes given stacker + * + * @param $stackerName Name of the stack + * @param $forceReInit Force re-initialization + * @return void + * @throws UnsupportedOperationException This method is not (and maybe never will be) supported + */ + public function initStack ($stackerName, $forceReInit = FALSE) { + throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION); + } + + /** + * Initializes all stacks + * + * @return void + * @throws UnsupportedOperationException This method is not (and maybe never will be) supported + */ + public function initStacks (array $stacks, $forceReInit = FALSE) { + throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION); + } + + /** + * Checks whether the given stack is initialized (set in array $stackers) + * + * @param $stackerName Name of the stack + * @return $isInitialized Whether the stack is initialized + * @throws UnsupportedOperationException This method is not (and maybe never will be) supported + */ + public function isStackInitialized ($stackerName) { + throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION); + } + + /** + * Getter for size of given stack (array count) + * + * @param $stackerName Name of the stack + * @return $count Size of stack (array count) + */ + public function getStackCount ($stackerName) { + // Now, simply return the found count value, this must be up-to-date then! + return $this->getCounter(); + } } // [EOF]