*/
const STACK_MAGIC = 'STACKv0.1';
- /**
- * Separator for header data
- */
- const SEPARATOR_HEADER_DATA = 0x01;
-
- /**
- * Separator header->entries
- */
- const SEPARATOR_HEADER_ENTRIES = 0x02;
-
- /**
- * Separator hash->name
- */
- const SEPARATOR_HASH_NAME = 0x03;
-
- /**
- * Length of name
- */
- const LENGTH_NAME = 10;
-
- /**
- * Length of count
- */
- const LENGTH_COUNT = 20;
-
- /**
- * Length of position
- */
- const LENGTH_POSITION = 20;
-
- /**
- * Counter for total entries
- */
- private $totalEntries = 0;
-
- /**
- * Current seek position
- */
- private $seekPosition = 0;
-
- /**
- * Size of header
- */
- private $headerSize = 0;
-
- /**
- * File header
- */
- private $header = array();
-
- /**
- * Seek positions for gaps ("fragmentation")
- */
- private $gaps = array();
-
- /**
- * Seek positions for damaged entries (e.g. mismatching hash sum, ...)
- */
- private $damagedEntries = array();
-
/**
* Protected constructor
*
parent::__construct($className);
// Calculate header size
- $this->headerSize = (
+ $this->setHeaderSize(
strlen(self::STACK_MAGIC) +
strlen(self::SEPARATOR_HEADER_DATA) +
self::LENGTH_COUNT +
$this->initCountersGapsArray();
}
- /**
- * Initializes counter for valid entries, arrays for damaged entries and
- * an array for gap seek positions. If you call this method on your own,
- * please re-analyze the file structure. So you are better to call
- * analyzeStackFile() instead of this method.
- *
- * @return void
- */
- private function initCountersGapsArray () {
- // Init counter and seek position
- $this->setCounter(0);
- $this->setSeekPosition(0);
-
- // Init arrays
- $this->gaps = array();
- $this->damagedEntries = array();
- }
-
- /**
- * Getter for total entries
- *
- * @return $totalEntries Total entries in this stack
- */
- private final function getCounter () {
- // Get it
- return $this->totalEntries;
- }
-
- /**
- * Setter for total entries
- *
- * @param $totalEntries Total entries in this stack
- * @return void
- */
- private final function setCounter ($counter) {
- // Set it
- $this->totalEntries = $counter;
- }
-
- /**
- * Increment counter
- *
- * @return void
- */
- private final function incrementCounter () {
- // Get it
- $this->totalEntries++;
- }
-
- /**
- * Getter for seek position
- *
- * @return $seekPosition Current seek position (stored here in object)
- */
- private final function getSeekPosition () {
- // Get it
- return $this->seekPosition;
- }
-
- /**
- * Setter for seek position
- *
- * @param $seekPosition Current seek position (stored here in object)
- * @return void
- */
- private final function setSeekPosition ($seekPosition) {
- // And set it
- $this->seekPosition = $seekPosition;
- }
-
- /**
- * Updates seekPosition attribute from file to avoid to much access on file.
- *
- * @return void
- */
- private function updateSeekPosition () {
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
-
- // Get key (= seek position)
- $seekPosition = $this->getIteratorInstance()->key();
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Setting seekPosition=%s', __METHOD__, __LINE__, $seekPosition));
-
- // And set it here
- $this->setSeekPosition($seekPosition);
-
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
- }
-
/**
* Reads the file header
*
* @return void
+ * @todo To hard assertions here, better rewrite them to exceptions
*/
- private function readFileHeader () {
+ public function readFileHeader () {
//* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
// First rewind to beginning as the header sits at the beginning ...
$this->getIteratorInstance()->rewind();
// Then read it (see constructor for calculation)
- $data = $this->getIteratorInstance()->read($this->headerSize);
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Read %d bytes (%d wanted).', __METHOD__, __LINE__, strlen($data), $this->headerSize));
+ $data = $this->getIteratorInstance()->read($this->getHeaderSize());
+ //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Read %d bytes (%d wanted).', __METHOD__, __LINE__, strlen($data), $this->getHeaderSize()));
// Have all requested bytes been read?
- assert(strlen($data) == $this->headerSize);
+ assert(strlen($data) == $this->getHeaderSize());
//* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
// Last character must be the separator
/*
* Now split it:
*
- * 0 => Magic
- * 1 => Total entries
- * 2 => Current seek position
+ * 0 => magic
+ * 1 => total entries
+ * 2 => current seek position
*/
- $this->header = explode(chr(self::SEPARATOR_HEADER_DATA), $data);
+ $header = explode(chr(self::SEPARATOR_HEADER_DATA), $data);
+
+ // Set header here
+ $this->setHeader($header);
// Check if the array has only 3 elements
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] header(%d)=%s', __METHOD__, __LINE__, count($this->header), print_r($this->header, TRUE)));
- assert(count($this->header) == 3);
+ //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] header(%d)=%s', __METHOD__, __LINE__, count($header), print_r($header, TRUE)));
+ assert(count($header) == 3);
//* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
// Check magic
- assert($this->header[0] == self::STACK_MAGIC);
+ assert($header[0] == self::STACK_MAGIC);
//* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
// Check length of count and seek position
- assert(strlen($this->header[1]) == self::LENGTH_COUNT);
+ assert(strlen($header[1]) == self::LENGTH_COUNT);
//* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
- assert(strlen($this->header[2]) == self::LENGTH_POSITION);
+ assert(strlen($header[2]) == self::LENGTH_POSITION);
//* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
// Decode count and seek position
- $this->header[1] = hex2bin($this->header[1]);
- $this->header[2] = hex2bin($this->header[2]);
+ $header[1] = hex2bin($header[1]);
+ $header[2] = hex2bin($header[2]);
//* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
}
- /**
- * Checks whether the file header is initialized
- *
- * @return $isInitialized Whether the file header is initialized
- */
- private function isFileHeaderInitialized () {
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
- // Default is not initialized
- $isInitialized = FALSE;
-
- // Is the file initialized?
- if ($this->isFileInitialized()) {
- // Some bytes has been written, so rewind to start of it.
- $rewindStatus = $this->getIteratorInstance()->rewind();
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] rewindStatus=%s', __METHOD__, __LINE__, $rewindStatus));
-
- // Is the rewind() call successfull?
- if ($rewindStatus != 1) {
- // Something bad happened
- self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Could not rewind().', __METHOD__, __LINE__));
- } // END - if
-
- // Read file header
- $this->readFileHeader();
-
- // The above method does already check the header
- $isInitialized = TRUE;
- } // END - if
-
- // Return result
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] isInitialized=%d - EXIT!', __METHOD__, __LINE__, intval($isInitialized)));
- return $isInitialized;
- }
-
- /**
- * Checks whether the file-based stack has been initialized
- *
- * @return $isInitialized Whether the file's size is zero
- */
- private function isFileInitialized () {
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
-
- // Get it from iterator which holds the pointer instance. If FALSE is returned
- $fileSize = $this->getIteratorInstance()->size();
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] fileSize=%s', __METHOD__, __LINE__, $fileSize));
-
- /*
- * The returned file size should not be FALSE or NULL as this means
- * that the pointer class does not work correctly.
- */
- assert(is_int($fileSize));
-
- // Is more than 0 returned?
- $isInitialized = ($fileSize > 0);
-
- // Return result
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] isInitialized=%d - EXIT!', __METHOD__, __LINE__, intval($isInitialized)));
- return $isInitialized;
- }
-
- /**
- * Creates the file-stack's header
- *
- * @return void
- */
- private function createFileHeader () {
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
- // The file's header should not be initialized here
- assert(!$this->isFileHeaderInitialized());
-
- // Simple flush file header which will create it.
- $this->flushFileHeader();
-
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!!', __METHOD__, __LINE__));
- }
-
/**
* Flushes the file header
*
* @return void
*/
- private function flushFileHeader () {
+ public function flushFileHeader () {
//* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
// Put all informations together
);
// Write it to disk (header is always at seek position 0)
- $this->writeData(0, $header);
-
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
- }
-
- /**
- * Writes data at given position
- *
- * @param $seekPosition Seek position
- * @param $data Data to be written
- * @return void
- */
- private function writeData ($seekPosition, $data) {
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] seekPosition=%s,data()=%s - CALLED!', __METHOD__, __LINE__, $seekPosition, strlen($data)));
-
- // Write data at given position
- $this->getIteratorInstance()->writeAtPosition($seekPosition, $data);
-
- // Update seek position
- $this->updateSeekPosition();
-
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
- }
-
- /**
- * Pre-allocates file (if enabled) with some space for later faster write access.
- *
- * @return void
- */
- private function preAllocateFile () {
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
-
- // Is it enabled?
- if ($this->getConfigInstance()->getConfigEntry('file_stack_pre_allocate_enabled') != 'Y') {
- // Not enabled
- self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Not pre-allocating stack file.', __METHOD__, __LINE__));
-
- // Don't continue here.
- return;
- } // END - if
-
- // Message to user
- self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Pre-allocating stack file ...', __METHOD__, __LINE__));
-
- /*
- * Calculate minimum length for one entry:
- * minimum length = hash length + separator + name + minimum entry size = ?? + 1 + 10 + 1 = ??
- */
- $minLengthEntry = self::getHashLength() + strlen(self::SEPARATOR_HASH_NAME) + self::LENGTH_NAME + 1;
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] minLengthEntry=%s', __METHOD__, __LINE__, $minLengthEntry));
-
- // Calulcate seek position
- $seekPosition = $minLengthEntry * $this->getConfigInstance()->getConfigEntry('file_stack_pre_allocate_count');
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] seekPosition=%s', __METHOD__, __LINE__, $seekPosition));
-
- // Now simply write a NUL there. This will pre-allocate the file.
- $this->writeData($seekPosition, chr(0));
-
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
- }
-
- /**
- * Analyzes entries in stack file. This will count all found (and valid)
- * entries, mark invalid as damaged and count gaps ("fragmentation"). If
- * only gaps are found, the file is considered as "virgin" (no entries).
- *
- * @return void
- */
- private function analyzeStackFile () {
- //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
-
- // Make sure the file is initialized
- assert($this->isFileInitialized());
-
- // Init counters and gaps array
- $this->initCountersGapsArray();
-
- // Output message (as this may take some time)
- self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Analyzing file structure ... (this may take some time)', __METHOD__, __LINE__));
+ $this->writeData(0, $header, FALSE);
//* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
}
* @param $fileName File name of this stack
* @param $type Type of this stack (e.g. url_source for URL sources)
* @return void
+ * @todo Currently the stack file is not cached, please implement a memory-handling class and if enough RAM is found, cache the whole stack file.
*/
protected function initFileStack ($fileName, $type) {
- // Get a file i/o pointer instance for stack file
- $pointerInstance = ObjectFactory::createObjectByConfiguredName('file_raw_input_output_class', array($fileName));
+ // Get a stack file instance
+ $fileInstance = ObjectFactory::createObjectByConfiguredName('stack_file_class', array($fileName, $this));
// Get iterator instance
- $iteratorInstance = ObjectFactory::createObjectByConfiguredName('file_io_iterator_class', array($pointerInstance));
+ $iteratorInstance = ObjectFactory::createObjectByConfiguredName('file_iterator_class', array($fileInstance));
// Is the instance implementing the right interface?
assert($iteratorInstance instanceof SeekableWritableFileIterator);
$this->createFileHeader();
// And pre-allocate a bit
- $this->preAllocateFile();
+ $this->preAllocateFile('file_stack');
} // END - if
// Load the file header
$this->readFileHeader();
// Count all entries in file
- $this->analyzeStackFile();
+ $this->getIteratorInstance()->analyzeFile();
/*
* Get stack index instance. This can be used for faster
throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
}
+ /**
+ * Close a file source and set it's instance to null and the file name
+ * to empty.
+ *
+ * @return void
+ * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
+ */
+ public function closeFile () {
+ throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
+ }
+
+ /**
+ * Determines whether the EOF has been reached
+ *
+ * @return $isEndOfFileReached Whether the EOF has been reached
+ * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
+ */
+ public function isEndOfFileReached () {
+ throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
+ }
+
+ /**
+ * Getter for file name
+ *
+ * @return $fileName The current file name
+ * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
+ */
+ public function getFileName () {
+ throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
+ }
+
/**
* Getter for size of given stack (array count)
*
// Now, simply return the found count value, this must be up-to-date then!
return $this->getCounter();
}
+
+ /**
+ * Calculates minimum length for one entry/block
+ *
+ * @return $length Minimum length for one entry/block
+ */
+ public function calculateMinimumBlockLength () {
+ // Calulcate it
+ $length = self::getHashLength() + strlen(chr(self::SEPARATOR_HASH_NAME)) + self::LENGTH_NAME + 1 + strlen(self::getBlockSeparator());
+
+ // Return it
+ return $length;
+ }
}
// [EOF]
projects / core.git / blobdiff