X-Git-Url: https://git.mxchange.org/?p=core.git;a=blobdiff_plain;f=inc%2Fclasses%2Fmain%2Fstacker%2Ffile%2Fclass_BaseFileStack.php;h=d16068d32d80ef3245a390e2f8ba0a626de8b49b;hp=70bd7006f5c1c345e273f59790c256470af3a0f8;hb=4af01023fc4b9ffc4c7174264dbff53966aecc91;hpb=79968ab0f673b50f09f35aed9bdc8112558e1553 diff --git a/inc/classes/main/stacker/file/class_BaseFileStack.php b/inc/classes/main/stacker/file/class_BaseFileStack.php index 70bd7006..d16068d3 100644 --- a/inc/classes/main/stacker/file/class_BaseFileStack.php +++ b/inc/classes/main/stacker/file/class_BaseFileStack.php @@ -27,41 +27,6 @@ class BaseFileStack extends BaseStacker { */ const STACK_MAGIC = 'STACKv0.1'; - /** - * Separator magic->count - */ - const SEPARATOR_MAGIC_COUNT = 0x00; - - /** - * Separator position->entries - */ - const SEPARATOR_SEEK_POS_ENTRIES = 0xff; - - /** - * Separator hash->name - */ - const SEPARATOR_HASH_NAME = 0x05; - - /** - * Length of count - */ - const COUNT_LENGTH = 20; - - /** - * Length of position - */ - const COUNT_POSITION = 20; - - /** - * Counter for total entries - */ - private $totalEntries = 0; - - /** - * Current seek position - */ - private $seekPosition = 0; - /** * Protected constructor * @@ -71,123 +36,83 @@ class BaseFileStack extends BaseStacker { protected function __construct ($className) { // Call parent constructor parent::__construct($className); - } - - /** - * Getter for total entries - * - * @return $totalEntries Total entries in this stack - */ - private function getCounter () { - // Get it - return $this->totalEntries; - } - /** - * Increment counter - * - * @return void - */ - private function incrementCounter () { - // Get it - $this->totalEntries++; - } + // Calculate header size + $this->setHeaderSize( + strlen(self::STACK_MAGIC) + + strlen(self::SEPARATOR_HEADER_DATA) + + self::LENGTH_COUNT + + strlen(self::SEPARATOR_HEADER_DATA) + + self::LENGTH_POSITION + + strlen(self::SEPARATOR_HEADER_ENTRIES) + ); - /** - * Getter for seek position - * - * @return $seekPosition Current seek position (stored here in object) - */ - private function getSeekPosition () { - // Get it - return $this->seekPosition; + // Init counters and gaps array + $this->initCountersGapsArray(); } /** - * Setter for seek position + * Reads the file header * - * @param $seekPosition Current seek position (stored here in object) * @return void */ - private function setSeekPosition ($seekPosition) { - // And set it - $this->seekPosition = $seekPosition; - } + protected function readFileHeader () { + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__)); - /** - * Updates seekPosition attribute from file to avoid to much access on file. - * - * @return void - */ - private function updateSeekPosition () { - // Get key (= seek position) - $seekPosition = $this->getIteratorInstance()->key(); + // First rewind to beginning as the header sits at the beginning ... + $this->getIteratorInstance()->rewind(); - // And set it here - $this->setSeekPosition($seekPosition); - } + // Then read it (see constructor for calculation) + $data = $this->getIteratorInstance()->read($this->getHeaderSize()); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Read %d bytes (%d wanted).', __METHOD__, __LINE__, strlen($data), $this->getHeaderSize())); - /** - * Checks whether the file header is initialized - * - * @return $isInitialized Whether the file header is initialized - */ - private function isFileHeaderInitialized () { - // Default is not initialized - $isInitialized = FALSE; + // Have all requested bytes been read? + assert(strlen($data) == $this->getHeaderSize()); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__)); - // Is the file initialized? - if ($this->isFileInitialized()) { - // Some bytes has been written, so rewind to start of it. - $this->getIteratorInstance()->rewind(); + // Last character must be the separator + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] data(-1)=%s', __METHOD__, __LINE__, dechex(ord(substr($data, -1, 1))))); + assert(substr($data, -1, 1) == chr(self::SEPARATOR_HEADER_ENTRIES)); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__)); - // Read file header - $this->readFileHeader(); - } // END - if + // Okay, then remove it + $data = substr($data, 0, -1); - // Return result - return $isInitialized; - } - - /** - * Checks whether the file-based stack has been initialized - * - * @return $isInitialized Whether the file's size is zero - */ - private function isFileInitialized () { - // Default is not initialized - $isInitialized = FALSE; - - // Get it from iterator which holds the pointer instance. If FALSE is returned - $fileSize = $this->getIteratorInstance()->size(); + // And update seek position + $this->updateSeekPosition(); /* - * The returned file size should not be FALSE or NULL as this means - * that the pointer class does not work correctly. + * Now split it: + * + * 0 => magic + * 1 => total entries + * 2 => current seek position */ - assert(is_int($fileSize)); + $header = explode(chr(self::SEPARATOR_HEADER_DATA), $data); - // Is more than 0 returned? - if ($fileSize > 0) { - // So is the header written? - $isInitialized = $this->getIteratorInstance()->isHeaderInitialized(); - } // END - if + // Set header here + $this->setHeader($header); - // Return result - return $isInitialized; - } + // Check if the array has only 3 elements + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] header(%d)=%s', __METHOD__, __LINE__, count($header), print_r($header, TRUE))); + assert(count($header) == 3); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__)); - /** - * Creates the file-stack's header - * - * @return void - */ - private function createFileHeader () { - // The file's header should not be initialized here - assert(!$this->isFileHeaderInitialized()); + // Check magic + assert($header[0] == self::STACK_MAGIC); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__)); + + // Check length of count and seek position + assert(strlen($header[1]) == self::LENGTH_COUNT); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__)); + assert(strlen($header[2]) == self::LENGTH_POSITION); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__)); - // Flush file header - $this->flushFileHeader(); + // Decode count and seek position + $header[1] = hex2bin($header[1]); + $header[2] = hex2bin($header[2]); + + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__)); } /** @@ -195,44 +120,87 @@ class BaseFileStack extends BaseStacker { * * @return void */ - private function flushFileHeader () { + protected function flushFileHeader () { + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__)); + // Put all informations together - $header = sprintf('%s%s%s%s%s', + $header = sprintf('%s%s%s%s%s%s', // Magic self::STACK_MAGIC, // Separator magic<->count - chr(self::SEPARATOR_MAGIC_COUNT), + chr(self::SEPARATOR_HEADER_DATA), // Total entries (will be zero) and pad it to 20 chars - str_pad($this->dec2hex($this->getCounter()), self::COUNT_LENGTH, '0', STR_PAD_LEFT), + str_pad($this->dec2hex($this->getCounter()), self::LENGTH_COUNT, '0', STR_PAD_LEFT), + + // Separator count<->seek position + chr(self::SEPARATOR_HEADER_DATA), // Position (will be zero) - str_pad($this->dec2hex(0, 2), self::COUNT_POSITION, '0', STR_PAD_LEFT), + str_pad($this->dec2hex($this->getSeekPosition(), 2), self::LENGTH_POSITION, '0', STR_PAD_LEFT), // Separator position<->entries - chr(self::SEPARATOR_SEEK_POS_ENTRIES) + chr(self::SEPARATOR_HEADER_ENTRIES) ); // Write it to disk (header is always at seek position 0) - $this->getIteratorInstance()->writeAtPosition(0, $header); + $this->writeData(0, $header, FALSE); - // Update seek position - $this->updateSeekPosition(); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__)); + } + + /** + * Analyzes entries in stack file. This will count all found (and valid) + * entries, mark invalid as damaged and count gaps ("fragmentation"). If + * only gaps are found, the file is considered as "virgin" (no entries). + * + * @return void + */ + private function analyzeFile () { + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__)); + + // Make sure the file is initialized + assert($this->isFileInitialized()); + + // Init counters and gaps array + $this->initCountersGapsArray(); + + // Output message (as this may take some time) + self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Analyzing file structure ... (this may take some time)', __METHOD__, __LINE__)); + + // First rewind to the begining + $this->getIteratorInstance()->rewind(); + + // Then try to load all entries + while ($this->getIteratorInstance()->valid()) { + // Go to next entry + $this->getIteratorInstance()->next(); + + // Get current entry + $current = $this->getIteratorInstance()->current(); + + // Simply output it + self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] current(%s)=%s', __METHOD__, __LINE__, strlen($current), print_r($current, TRUE))); + } // END - while + + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__)); } /** * Initializes this file-based stack. * * @param $fileName File name of this stack + * @param $type Type of this stack (e.g. url_source for URL sources) * @return void + * @todo Currently the stack file is not cached, please implement a memory-handling class and if enough RAM is found, cache the whole stack file. */ - protected function initFileStack ($fileName) { - // Get a file i/o pointer instance - $pointerInstance = ObjectFactory::createObjectByConfiguredName('file_raw_input_output_class', array($fileName)); + protected function initFileStack ($fileName, $type) { + // Get a stack file instance + $fileInstance = ObjectFactory::createObjectByConfiguredName('stack_file_class', array($fileName)); // Get iterator instance - $iteratorInstance = ObjectFactory::createObjectByConfiguredName('file_io_iterator_class', array($pointerInstance)); + $iteratorInstance = ObjectFactory::createObjectByConfiguredName('file_io_iterator_class', array($fileInstance, $this)); // Is the instance implementing the right interface? assert($iteratorInstance instanceof SeekableWritableFileIterator); @@ -241,72 +209,28 @@ class BaseFileStack extends BaseStacker { $this->setIteratorInstance($iteratorInstance); // Is the file's header initialized? - if ($this->isFileHeaderInitialized()) { - // Then load it - $this->loadFileHeader(); - } else { + if (!$this->isFileHeaderInitialized()) { // No, then create it (which may pre-allocate the stack) $this->createFileHeader(); // And pre-allocate a bit - $this->preAllocateFile(); - } - } - - /** - * Initializes given stacker - * - * @param $stackerName Name of the stack - * @param $forceReInit Force re-initialization - * @return void - * @throws AlreadyInitializedStackerException If the stack is already initialized - */ - public function initStack ($stackerName, $forceReInit = FALSE) { - // Is the stack already initialized? - if (($forceReInit === FALSE) && ($this->isStackInitialized($stackerName))) { - // Then throw the exception - throw new AlreadyInitializedStackerException(array($this, $stackerName, $forceReInit), self::EXCEPTION_STACKER_ALREADY_INITIALIZED); + $this->preAllocateFile('file_stack'); } // END - if - // Initialize the given stack - $this->partialStub('stackerName=' . $stackerName . ',forceReInit=' . intval($forceReInit)); - } + // Load the file header + $this->readFileHeader(); - /** - * Checks whether the given stack is initialized (set in array $stackers) - * - * @param $stackerName Name of the stack - * @return $isInitialized Whether the stack is initialized - */ - public function isStackInitialized ($stackerName) { - // Is is there? - $this->partialStub('stackerName=' . $stackerName); - $isInitialized = TRUE; - - // Return result - return $isInitialized; - } - - /** - * Getter for size of given stack (array count) - * - * @param $stackerName Name of the stack - * @return $count Size of stack (array count) - * @throws NoStackerException If given stack is missing - */ - public function getStackCount ($stackerName) { - // Is the stack not yet initialized? - if (!$this->isStackInitialized($stackerName)) { - // Throw an exception - throw new NoStackerException(array($this, $stackerName), self::EXCEPTION_NO_STACKER_FOUND); - } // END - if + // Count all entries in file + $this->analyzeFile(); - // Now, count the array of entries - $this->partialStub('stackerName=' . $stackerName); - $count = 0; + /* + * Get stack index instance. This can be used for faster + * "defragmentation" and startup. + */ + $indexInstance = FileStackIndexFactory::createFileStackIndexInstance($fileName, $type); - // Return result - return $count; + // And set it here + $this->setIndexInstance($indexInstance); } /** @@ -315,17 +239,14 @@ class BaseFileStack extends BaseStacker { * @param $stackerName Name of the stack * @param $value Value to add to this stacker * @return void - * @throws FullStackerException Thrown if the stack is full + * @throws FullStackerException If the stack is full */ protected function addValue ($stackerName, $value) { - // Is the stack not yet initialized or full? - if (!$this->isStackInitialized($stackerName)) { - // Then do it here - $this->initStack($stackerName); - } elseif ($this->isStackFull($stackerName)) { + // Do some tests + if ($this->isStackFull($stackerName)) { // Stacker is full throw new FullStackerException(array($this, $stackerName, $value), self::EXCEPTION_STACKER_IS_FULL); - } + } // END - if // Now add the value to the stack $this->partialStub('stackerName=' . $stackerName . ',value[]=' . gettype($value)); @@ -336,18 +257,14 @@ class BaseFileStack extends BaseStacker { * * @param $stackerName Name of the stack * @return $value Value of last added value - * @throws NoStackerException If the named stacker was not found - * @throws EmptyStackerException If the named stacker is empty + * @throws EmptyStackerException If the stack is empty */ protected function getLastValue ($stackerName) { // Is the stack not yet initialized or full? - if (!$this->isStackInitialized($stackerName)) { - // Throw an exception - throw new NoStackerException(array($this, $stackerName), self::EXCEPTION_NO_STACKER_FOUND); - } elseif ($this->isStackEmpty($stackerName)) { + if ($this->isStackEmpty($stackerName)) { // Throw an exception throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY); - } + } // END - if // Now get the last value $this->partialStub('stackerName=' . $stackerName); @@ -362,18 +279,14 @@ class BaseFileStack extends BaseStacker { * * @param $stackerName Name of the stack * @return $value Value of last added value - * @throws NoStackerException If the named stacker was not found - * @throws EmptyStackerException If the named stacker is empty + * @throws EmptyStackerException If the stack is empty */ protected function getFirstValue ($stackerName) { // Is the stack not yet initialized or full? - if (!$this->isStackInitialized($stackerName)) { - // Throw an exception - throw new NoStackerException(array($this, $stackerName), self::EXCEPTION_NO_STACKER_FOUND); - } elseif ($this->isStackEmpty($stackerName)) { + if ($this->isStackEmpty($stackerName)) { // Throw an exception throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY); - } + } // END - if // Now get the first value $this->partialStub('stackerName=' . $stackerName); @@ -388,18 +301,14 @@ class BaseFileStack extends BaseStacker { * * @param $stackerName Name of the stack * @return $value Value "poped" from array - * @throws NoStackerException If the named stacker was not found - * @throws EmptyStackerException If the named stacker is empty + * @throws EmptyStackerException If the stack is empty */ protected function popLast ($stackerName) { // Is the stack not yet initialized or full? - if (!$this->isStackInitialized($stackerName)) { - // Throw an exception - throw new NoStackerException(array($this, $stackerName), self::EXCEPTION_NO_STACKER_FOUND); - } elseif ($this->isStackEmpty($stackerName)) { + if ($this->isStackEmpty($stackerName)) { // Throw an exception throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY); - } + } // END - if // Now, remove the last entry, we don't care about the return value here, see elseif() block above $this->partialStub('stackerName=' . $stackerName); @@ -411,23 +320,106 @@ class BaseFileStack extends BaseStacker { * * @param $stackerName Name of the stack * @return $value Value "shifted" from array - * @throws NoStackerException If the named stacker was not found * @throws EmptyStackerException If the named stacker is empty */ protected function popFirst ($stackerName) { // Is the stack not yet initialized or full? - if (!$this->isStackInitialized($stackerName)) { - // Throw an exception - throw new NoStackerException(array($this, $stackerName), self::EXCEPTION_NO_STACKER_FOUND); - } elseif ($this->isStackEmpty($stackerName)) { + if ($this->isStackEmpty($stackerName)) { // Throw an exception throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY); - } + } // END - if // Now, remove the last entry, we don't care about the return value here, see elseif() block above $this->partialStub('stackerName=' . $stackerName); return NULL; } + + /** + * Checks whether the given stack is full + * + * @param $stackerName Name of the stack + * @return $isFull Whether the stack is full + */ + protected function isStackFull ($stackerName) { + // File-based stacks will only run full if the disk space is low. + // @TODO Please implement this, returning FALSE + $isFull = FALSE; + + // Return result + return $isFull; + } + + /** + * Checks whether the given stack is empty + * + * @param $stackerName Name of the stack + * @return $isEmpty Whether the stack is empty + * @throws NoStackerException If given stack is missing + */ + public function isStackEmpty ($stackerName) { + // So, is the stack empty? + $isEmpty = (($this->getStackCount($stackerName)) == 0); + + // Return result + return $isEmpty; + } + + /** + * Initializes given stacker + * + * @param $stackerName Name of the stack + * @param $forceReInit Force re-initialization + * @return void + * @throws UnsupportedOperationException This method is not (and maybe never will be) supported + */ + public function initStack ($stackerName, $forceReInit = FALSE) { + throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION); + } + + /** + * Initializes all stacks + * + * @return void + * @throws UnsupportedOperationException This method is not (and maybe never will be) supported + */ + public function initStacks (array $stacks, $forceReInit = FALSE) { + throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION); + } + + /** + * Checks whether the given stack is initialized (set in array $stackers) + * + * @param $stackerName Name of the stack + * @return $isInitialized Whether the stack is initialized + * @throws UnsupportedOperationException This method is not (and maybe never will be) supported + */ + public function isStackInitialized ($stackerName) { + throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION); + } + + /** + * Getter for size of given stack (array count) + * + * @param $stackerName Name of the stack + * @return $count Size of stack (array count) + */ + public function getStackCount ($stackerName) { + // Now, simply return the found count value, this must be up-to-date then! + return $this->getCounter(); + } + + /** + * Calculates minimum length for one entry/block + * + * @return $length Minimum length for one entry/block + */ + public function caluclateMinimumBlockLength () { + // Calulcate it + $length = self::getHashLength() + strlen(chr(self::SEPARATOR_HASH_NAME)) + self::LENGTH_NAME + 1 + strlen(self::getBlockSeparator()); + + // Return it + return $length; + } } // [EOF]