3 * A general file-based stack class
5 * @author Roland Haeder <webmaster@ship-simu.org>
7 * @copyright Copyright (c) 2007, 2008 Roland Haeder, 2009 - 2013 Core Developer Team
8 * @license GNU GPL 3.0 or any newer version
9 * @link http://www.ship-simu.org
11 * This program is free software: you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation, either version 3 of the License, or
14 * (at your option) any later version.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with this program. If not, see <http://www.gnu.org/licenses/>.
24 class BaseFileStack extends BaseStacker {
26 * Magic for this stack
28 const STACK_MAGIC = 'STACKv0.1';
31 * Separator for header data
33 const SEPARATOR_HEADER_DATA = 0x01;
36 * Separator header->entries
38 const SEPARATOR_HEADER_ENTRIES = 0x02;
41 * Separator hash->name
43 const SEPARATOR_HASH_NAME = 0x03;
48 const LENGTH_NAME = 10;
53 const LENGTH_COUNT = 20;
58 const LENGTH_POSITION = 20;
61 * Counter for total entries
63 private $totalEntries = 0;
66 * Current seek position
68 private $seekPosition = 0;
73 private $headerSize = 0;
78 private $header = array();
81 * Seek positions for gaps ("fragmentation")
83 private $gaps = array();
86 * Seek positions for damaged entries (e.g. mismatching hash sum, ...)
88 private $damagedEntries = array();
91 * Protected constructor
93 * @param $className Name of the class
96 protected function __construct ($className) {
97 // Call parent constructor
98 parent::__construct($className);
100 // Calculate header size
101 $this->headerSize = (
102 strlen(self::STACK_MAGIC) +
103 strlen(self::SEPARATOR_HEADER_DATA) +
105 strlen(self::SEPARATOR_HEADER_DATA) +
106 self::LENGTH_POSITION +
107 strlen(self::SEPARATOR_HEADER_ENTRIES)
110 // Init counters and gaps array
111 $this->initCountersGapsArray();
115 * Initializes counter for valid entries, arrays for damaged entries and
116 * an array for gap seek positions. If you call this method on your own,
117 * please re-analyze the file structure. So you are better to call
118 * analyzeStackFile() instead of this method.
122 private function initCountersGapsArray () {
123 // Init counter and seek position
124 $this->setCounter(0);
125 $this->setSeekPosition(0);
128 $this->gaps = array();
129 $this->damagedEntries = array();
133 * Getter for total entries
135 * @return $totalEntries Total entries in this stack
137 private final function getCounter () {
139 return $this->totalEntries;
143 * Setter for total entries
145 * @param $totalEntries Total entries in this stack
148 private final function setCounter ($counter) {
150 $this->totalEntries = $counter;
158 private final function incrementCounter () {
160 $this->totalEntries++;
164 * Getter for seek position
166 * @return $seekPosition Current seek position (stored here in object)
168 private final function getSeekPosition () {
170 return $this->seekPosition;
174 * Setter for seek position
176 * @param $seekPosition Current seek position (stored here in object)
179 private final function setSeekPosition ($seekPosition) {
181 $this->seekPosition = $seekPosition;
185 * Updates seekPosition attribute from file to avoid to much access on file.
189 private function updateSeekPosition () {
190 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
192 // Get key (= seek position)
193 $seekPosition = $this->getIteratorInstance()->key();
194 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Setting seekPosition=%s', __METHOD__, __LINE__, $seekPosition));
197 $this->setSeekPosition($seekPosition);
199 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
203 * Reads the file header
207 private function readFileHeader () {
208 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
210 // First rewind to beginning as the header sits at the beginning ...
211 $this->getIteratorInstance()->rewind();
213 // Then read it (see constructor for calculation)
214 $data = $this->getIteratorInstance()->read($this->headerSize);
215 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Read %d bytes (%d wanted).', __METHOD__, __LINE__, strlen($data), $this->headerSize));
217 // Have all requested bytes been read?
218 assert(strlen($data) == $this->headerSize);
219 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
221 // Last character must be the separator
222 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] data(-1)=%s', __METHOD__, __LINE__, dechex(ord(substr($data, -1, 1)))));
223 assert(substr($data, -1, 1) == chr(self::SEPARATOR_HEADER_ENTRIES));
224 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
226 // Okay, then remove it
227 $data = substr($data, 0, -1);
229 // And update seek position
230 $this->updateSeekPosition();
237 * 2 => Current seek position
239 $this->header = explode(chr(self::SEPARATOR_HEADER_DATA), $data);
241 // Check if the array has only 3 elements
242 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] header(%d)=%s', __METHOD__, __LINE__, count($this->header), print_r($this->header, TRUE)));
243 assert(count($this->header) == 3);
244 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
247 assert($this->header[0] == self::STACK_MAGIC);
248 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
250 // Check length of count and seek position
251 assert(strlen($this->header[1]) == self::LENGTH_COUNT);
252 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
253 assert(strlen($this->header[2]) == self::LENGTH_POSITION);
254 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
256 // Decode count and seek position
257 $this->header[1] = hex2bin($this->header[1]);
258 $this->header[2] = hex2bin($this->header[2]);
260 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
264 * Checks whether the file header is initialized
266 * @return $isInitialized Whether the file header is initialized
268 private function isFileHeaderInitialized () {
269 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
270 // Default is not initialized
271 $isInitialized = FALSE;
273 // Is the file initialized?
274 if ($this->isFileInitialized()) {
275 // Some bytes has been written, so rewind to start of it.
276 $rewindStatus = $this->getIteratorInstance()->rewind();
277 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] rewindStatus=%s', __METHOD__, __LINE__, $rewindStatus));
279 // Is the rewind() call successfull?
280 if ($rewindStatus != 1) {
281 // Something bad happened
282 self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Could not rewind().', __METHOD__, __LINE__));
286 $this->readFileHeader();
288 // The above method does already check the header
289 $isInitialized = TRUE;
293 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] isInitialized=%d - EXIT!', __METHOD__, __LINE__, intval($isInitialized)));
294 return $isInitialized;
298 * Checks whether the file-based stack has been initialized
300 * @return $isInitialized Whether the file's size is zero
302 private function isFileInitialized () {
303 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
305 // Get it from iterator which holds the pointer instance. If FALSE is returned
306 $fileSize = $this->getIteratorInstance()->size();
307 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] fileSize=%s', __METHOD__, __LINE__, $fileSize));
310 * The returned file size should not be FALSE or NULL as this means
311 * that the pointer class does not work correctly.
313 assert(is_int($fileSize));
315 // Is more than 0 returned?
316 $isInitialized = ($fileSize > 0);
319 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] isInitialized=%d - EXIT!', __METHOD__, __LINE__, intval($isInitialized)));
320 return $isInitialized;
324 * Creates the file-stack's header
328 private function createFileHeader () {
329 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
330 // The file's header should not be initialized here
331 assert(!$this->isFileHeaderInitialized());
333 // Simple flush file header which will create it.
334 $this->flushFileHeader();
336 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!!', __METHOD__, __LINE__));
340 * Flushes the file header
344 private function flushFileHeader () {
345 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
347 // Put all informations together
348 $header = sprintf('%s%s%s%s%s%s',
352 // Separator magic<->count
353 chr(self::SEPARATOR_HEADER_DATA),
355 // Total entries (will be zero) and pad it to 20 chars
356 str_pad($this->dec2hex($this->getCounter()), self::LENGTH_COUNT, '0', STR_PAD_LEFT),
358 // Separator count<->seek position
359 chr(self::SEPARATOR_HEADER_DATA),
361 // Position (will be zero)
362 str_pad($this->dec2hex($this->getSeekPosition(), 2), self::LENGTH_POSITION, '0', STR_PAD_LEFT),
364 // Separator position<->entries
365 chr(self::SEPARATOR_HEADER_ENTRIES)
368 // Write it to disk (header is always at seek position 0)
369 $this->writeData(0, $header);
371 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
375 * Writes data at given position
377 * @param $seekPosition Seek position
378 * @param $data Data to be written
381 private function writeData ($seekPosition, $data) {
382 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] seekPosition=%s,data()=%s - CALLED!', __METHOD__, __LINE__, $seekPosition, strlen($data)));
384 // Write data at given position
385 $this->getIteratorInstance()->writeAtPosition($seekPosition, $data);
387 // Update seek position
388 $this->updateSeekPosition();
390 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
394 * Pre-allocates file (if enabled) with some space for later faster write access.
398 private function preAllocateFile () {
399 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
402 if ($this->getConfigInstance()->getConfigEntry('file_stack_pre_allocate_enabled') != 'Y') {
404 self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Not pre-allocating stack file.', __METHOD__, __LINE__));
406 // Don't continue here.
411 self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Pre-allocating stack file ...', __METHOD__, __LINE__));
414 * Calculate minimum length for one entry:
415 * minimum length = hash length + separator + name + minimum entry size = ?? + 1 + 10 + 1 = ??
417 $minLengthEntry = self::getHashLength() + strlen(self::SEPARATOR_HASH_NAME) + self::LENGTH_NAME + 1;
418 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] minLengthEntry=%s', __METHOD__, __LINE__, $minLengthEntry));
420 // Calulcate seek position
421 $seekPosition = $minLengthEntry * $this->getConfigInstance()->getConfigEntry('file_stack_pre_allocate_count');
422 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] seekPosition=%s', __METHOD__, __LINE__, $seekPosition));
424 // Now simply write a NUL there. This will pre-allocate the file.
425 $this->writeData($seekPosition, chr(0));
427 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
431 * Analyzes entries in stack file. This will count all found (and valid)
432 * entries, mark invalid as damaged and count gaps ("fragmentation"). If
433 * only gaps are found, the file is considered as "virgin" (no entries).
437 private function analyzeStackFile () {
438 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
440 // Make sure the file is initialized
441 assert($this->isFileInitialized());
443 // Init counters and gaps array
444 $this->initCountersGapsArray();
446 // Output message (as this may take some time)
447 self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Analyzing file structure ... (this may take some time)', __METHOD__, __LINE__));
449 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
453 * Initializes this file-based stack.
455 * @param $fileName File name of this stack
456 * @param $type Type of this stack (e.g. url_source for URL sources)
459 protected function initFileStack ($fileName, $type) {
460 // Get a file i/o pointer instance for stack file
461 $pointerInstance = ObjectFactory::createObjectByConfiguredName('file_raw_input_output_class', array($fileName));
463 // Get iterator instance
464 $iteratorInstance = ObjectFactory::createObjectByConfiguredName('file_io_iterator_class', array($pointerInstance));
466 // Is the instance implementing the right interface?
467 assert($iteratorInstance instanceof SeekableWritableFileIterator);
470 $this->setIteratorInstance($iteratorInstance);
472 // Is the file's header initialized?
473 if (!$this->isFileHeaderInitialized()) {
474 // No, then create it (which may pre-allocate the stack)
475 $this->createFileHeader();
477 // And pre-allocate a bit
478 $this->preAllocateFile();
481 // Load the file header
482 $this->readFileHeader();
484 // Count all entries in file
485 $this->analyzeStackFile();
488 * Get stack index instance. This can be used for faster
489 * "defragmentation" and startup.
491 $indexInstance = FileStackIndexFactory::createFileStackIndexInstance($fileName, $type);
494 $this->setIndexInstance($indexInstance);
498 * Adds a value to given stack
500 * @param $stackerName Name of the stack
501 * @param $value Value to add to this stacker
503 * @throws FullStackerException If the stack is full
505 protected function addValue ($stackerName, $value) {
507 if ($this->isStackFull($stackerName)) {
509 throw new FullStackerException(array($this, $stackerName, $value), self::EXCEPTION_STACKER_IS_FULL);
512 // Now add the value to the stack
513 $this->partialStub('stackerName=' . $stackerName . ',value[]=' . gettype($value));
517 * Get last value from named stacker
519 * @param $stackerName Name of the stack
520 * @return $value Value of last added value
521 * @throws EmptyStackerException If the stack is empty
523 protected function getLastValue ($stackerName) {
524 // Is the stack not yet initialized or full?
525 if ($this->isStackEmpty($stackerName)) {
526 // Throw an exception
527 throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY);
530 // Now get the last value
531 $this->partialStub('stackerName=' . $stackerName);
539 * Get first value from named stacker
541 * @param $stackerName Name of the stack
542 * @return $value Value of last added value
543 * @throws EmptyStackerException If the stack is empty
545 protected function getFirstValue ($stackerName) {
546 // Is the stack not yet initialized or full?
547 if ($this->isStackEmpty($stackerName)) {
548 // Throw an exception
549 throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY);
552 // Now get the first value
553 $this->partialStub('stackerName=' . $stackerName);
561 * "Pops" last entry from stack
563 * @param $stackerName Name of the stack
564 * @return $value Value "poped" from array
565 * @throws EmptyStackerException If the stack is empty
567 protected function popLast ($stackerName) {
568 // Is the stack not yet initialized or full?
569 if ($this->isStackEmpty($stackerName)) {
570 // Throw an exception
571 throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY);
574 // Now, remove the last entry, we don't care about the return value here, see elseif() block above
575 $this->partialStub('stackerName=' . $stackerName);
580 * "Pops" first entry from stack
582 * @param $stackerName Name of the stack
583 * @return $value Value "shifted" from array
584 * @throws EmptyStackerException If the named stacker is empty
586 protected function popFirst ($stackerName) {
587 // Is the stack not yet initialized or full?
588 if ($this->isStackEmpty($stackerName)) {
589 // Throw an exception
590 throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY);
593 // Now, remove the last entry, we don't care about the return value here, see elseif() block above
594 $this->partialStub('stackerName=' . $stackerName);
599 * Checks whether the given stack is full
601 * @param $stackerName Name of the stack
602 * @return $isFull Whether the stack is full
604 protected function isStackFull ($stackerName) {
605 // File-based stacks will only run full if the disk space is low.
606 // @TODO Please implement this, returning FALSE
614 * Checks whether the given stack is empty
616 * @param $stackerName Name of the stack
617 * @return $isEmpty Whether the stack is empty
618 * @throws NoStackerException If given stack is missing
620 public function isStackEmpty ($stackerName) {
621 // So, is the stack empty?
622 $isEmpty = (($this->getStackCount($stackerName)) == 0);
629 * Initializes given stacker
631 * @param $stackerName Name of the stack
632 * @param $forceReInit Force re-initialization
634 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
636 public function initStack ($stackerName, $forceReInit = FALSE) {
637 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
641 * Initializes all stacks
644 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
646 public function initStacks (array $stacks, $forceReInit = FALSE) {
647 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
651 * Checks whether the given stack is initialized (set in array $stackers)
653 * @param $stackerName Name of the stack
654 * @return $isInitialized Whether the stack is initialized
655 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
657 public function isStackInitialized ($stackerName) {
658 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
662 * Getter for size of given stack (array count)
664 * @param $stackerName Name of the stack
665 * @return $count Size of stack (array count)
667 public function getStackCount ($stackerName) {
668 // Now, simply return the found count value, this must be up-to-date then!
669 return $this->getCounter();