3 * A general file-based stack class
5 * @author Roland Haeder <webmaster@ship-simu.org>
7 * @copyright Copyright (c) 2007, 2008 Roland Haeder, 2009 - 2013 Core Developer Team
8 * @license GNU GPL 3.0 or any newer version
9 * @link http://www.ship-simu.org
11 * This program is free software: you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation, either version 3 of the License, or
14 * (at your option) any later version.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with this program. If not, see <http://www.gnu.org/licenses/>.
24 class BaseFileStack extends BaseStacker {
26 * Magic for this stack
28 const STACK_MAGIC = 'STACKv0.1';
31 * Separator for header data
33 const SEPARATOR_HEADER_DATA = 0x01;
36 * Separator header->entries
38 const SEPARATOR_HEADER_ENTRIES = 0x02;
41 * Separator hash->name
43 const SEPARATOR_HASH_NAME = 0x03;
48 const LENGTH_NAME = 10;
53 const LENGTH_COUNT = 20;
58 const LENGTH_POSITION = 20;
61 * Counter for total entries
63 private $totalEntries = 0;
66 * Current seek position
68 private $seekPosition = 0;
73 private $headerSize = 0;
78 private $header = array();
81 * Seek positions for gaps ("fragmentation")
83 private $gaps = array();
86 * Seek positions for damaged entries (e.g. mismatching hash sum, ...)
88 private $damagedEntries = array();
91 * Protected constructor
93 * @param $className Name of the class
96 protected function __construct ($className) {
97 // Call parent constructor
98 parent::__construct($className);
100 // Calculate header size
101 $this->headerSize = (
102 strlen(self::STACK_MAGIC) +
103 strlen(self::SEPARATOR_HEADER_DATA) +
105 strlen(self::SEPARATOR_HEADER_DATA) +
106 self::LENGTH_POSITION +
107 strlen(self::SEPARATOR_HEADER_ENTRIES)
110 // Init counters and gaps array
111 $this->initCountersGapsArray();
115 * Initializes counter for valid entries, arrays for damaged entries and
116 * an array for gap seek positions. If you call this method on your own,
117 * please re-analyze the file structure. So you are better to call
118 * analyzeStackFile() instead of this method.
122 private function initCountersGapsArray () {
123 // Init counter and seek position
124 $this->setCounter(0);
125 $this->setSeekPosition(0);
128 $this->gaps = array();
129 $this->damagedEntries = array();
133 * Getter for total entries
135 * @return $totalEntries Total entries in this stack
137 private final function getCounter () {
139 return $this->totalEntries;
147 private final function incrementCounter () {
149 $this->totalEntries++;
153 * Getter for seek position
155 * @return $seekPosition Current seek position (stored here in object)
157 private final function getSeekPosition () {
159 return $this->seekPosition;
163 * Setter for seek position
165 * @param $seekPosition Current seek position (stored here in object)
168 private final function setSeekPosition ($seekPosition) {
170 $this->seekPosition = $seekPosition;
174 * Updates seekPosition attribute from file to avoid to much access on file.
178 private function updateSeekPosition () {
179 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
181 // Get key (= seek position)
182 $seekPosition = $this->getIteratorInstance()->key();
183 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Setting seekPosition=%s', __METHOD__, __LINE__, $seekPosition));
186 $this->setSeekPosition($seekPosition);
188 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
192 * Reads the file header
196 private function readFileHeader () {
197 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
199 // First rewind to beginning as the header sits at the beginning ...
200 $this->getIteratorInstance()->rewind();
202 // Then read it (see constructor for calculation)
203 $data = $this->getIteratorInstance()->read($this->headerSize);
204 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Read %d bytes (%d wanted).', __METHOD__, __LINE__, strlen($data), $this->headerSize));
206 // Have all requested bytes been read?
207 assert(strlen($data) == $this->headerSize);
208 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
210 // Last character must be the separator
211 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] data(-1)=%s', __METHOD__, __LINE__, dechex(ord(substr($data, -1, 1)))));
212 assert(substr($data, -1, 1) == chr(self::SEPARATOR_HEADER_ENTRIES));
213 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
215 // Okay, then remove it
216 $data = substr($data, 0, -1);
218 // And update seek position
219 $this->updateSeekPosition();
226 * 2 => Current seek position
228 $this->header = explode(chr(self::SEPARATOR_HEADER_DATA), $data);
230 // Check if the array has only 3 elements
231 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] header(%d)=%s', __METHOD__, __LINE__, count($this->header), print_r($this->header, TRUE)));
232 assert(count($this->header) == 3);
233 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
236 assert($this->header[0] == self::STACK_MAGIC);
237 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
239 // Check length of count and seek position
240 assert(strlen($this->header[1]) == self::LENGTH_COUNT);
241 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
242 assert(strlen($this->header[2]) == self::LENGTH_POSITION);
243 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
245 // Decode count and seek position
246 $this->header[1] = hex2bin($this->header[1]);
247 $this->header[2] = hex2bin($this->header[2]);
249 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
253 * Checks whether the file header is initialized
255 * @return $isInitialized Whether the file header is initialized
257 private function isFileHeaderInitialized () {
258 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
259 // Default is not initialized
260 $isInitialized = FALSE;
262 // Is the file initialized?
263 if ($this->isFileInitialized()) {
264 // Some bytes has been written, so rewind to start of it.
265 $rewindStatus = $this->getIteratorInstance()->rewind();
266 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] rewindStatus=%s', __METHOD__, __LINE__, $rewindStatus));
268 // Is the rewind() call successfull?
269 if ($rewindStatus != 1) {
270 // Something bad happened
271 self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Could not rewind().', __METHOD__, __LINE__));
275 $this->readFileHeader();
277 // The above method does already check the header
278 $isInitialized = TRUE;
282 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] isInitialized=%d - EXIT!', __METHOD__, __LINE__, intval($isInitialized)));
283 return $isInitialized;
287 * Checks whether the file-based stack has been initialized
289 * @return $isInitialized Whether the file's size is zero
291 private function isFileInitialized () {
292 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
294 // Get it from iterator which holds the pointer instance. If FALSE is returned
295 $fileSize = $this->getIteratorInstance()->size();
296 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] fileSize=%s', __METHOD__, __LINE__, $fileSize));
299 * The returned file size should not be FALSE or NULL as this means
300 * that the pointer class does not work correctly.
302 assert(is_int($fileSize));
304 // Is more than 0 returned?
305 $isInitialized = ($fileSize > 0);
308 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] isInitialized=%d - EXIT!', __METHOD__, __LINE__, intval($isInitialized)));
309 return $isInitialized;
313 * Creates the file-stack's header
317 private function createFileHeader () {
318 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
319 // The file's header should not be initialized here
320 assert(!$this->isFileHeaderInitialized());
322 // Simple flush file header which will create it.
323 $this->flushFileHeader();
325 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!!', __METHOD__, __LINE__));
329 * Flushes the file header
333 private function flushFileHeader () {
334 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
336 // Put all informations together
337 $header = sprintf('%s%s%s%s%s%s',
341 // Separator magic<->count
342 chr(self::SEPARATOR_HEADER_DATA),
344 // Total entries (will be zero) and pad it to 20 chars
345 str_pad($this->dec2hex($this->getCounter()), self::LENGTH_COUNT, '0', STR_PAD_LEFT),
347 // Separator count<->seek position
348 chr(self::SEPARATOR_HEADER_DATA),
350 // Position (will be zero)
351 str_pad($this->dec2hex($this->getSeekPosition(), 2), self::LENGTH_POSITION, '0', STR_PAD_LEFT),
353 // Separator position<->entries
354 chr(self::SEPARATOR_HEADER_ENTRIES)
357 // Write it to disk (header is always at seek position 0)
358 $this->writeData(0, $header);
360 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
364 * Writes data at given position
366 * @param $seekPosition Seek position
367 * @param $data Data to be written
370 private function writeData ($seekPosition, $data) {
371 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] seekPosition=%s,data()=%s - CALLED!', __METHOD__, __LINE__, $seekPosition, strlen($data)));
373 // Write data at given position
374 $this->getIteratorInstance()->writeAtPosition($seekPosition, $data);
376 // Update seek position
377 $this->updateSeekPosition();
379 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
383 * Pre-allocates file (if enabled) with some space for later faster write access.
387 private function preAllocateFile () {
388 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
391 if ($this->getConfigInstance()->getConfigEntry('file_stack_pre_allocate_enabled') != 'Y') {
393 self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Not pre-allocating stack file.', __METHOD__, __LINE__));
395 // Don't continue here.
400 self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Pre-allocating stack file ...', __METHOD__, __LINE__));
403 * Calculate minimum length for one entry:
404 * minimum length = hash length + separator + name + minimum entry size = ?? + 1 + 10 + 1 = ??
406 $minLengthEntry = self::getHashLength() + strlen(self::SEPARATOR_HASH_NAME) + self::LENGTH_NAME + 1;
407 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] minLengthEntry=%s', __METHOD__, __LINE__, $minLengthEntry));
409 // Calulcate seek position
410 $seekPosition = $minLengthEntry * $this->getConfigInstance()->getConfigEntry('file_stack_pre_allocate_count');
411 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] seekPosition=%s', __METHOD__, __LINE__, $seekPosition));
413 // Now simply write a NUL there. This will pre-allocate the file.
414 $this->writeData($seekPosition, chr(0));
416 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
420 * Analyzes entries in stack file. This will count all found (and valid)
421 * entries, mark invalid as damaged and count gaps ("fragmentation"). If
422 * only gaps are found, the file is considered as "virgin" (no entries).
426 private function analyzeStackFile () {
427 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
429 // Make sure the file is initialized
430 assert($this->isFileInitialized());
432 // Init counters and gaps array
433 $this->initCountersGapsArray();
435 // Output message (as this may take some time)
436 self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] Analyzing file structure ... (this may take some time)', __METHOD__, __LINE__));
438 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
442 * Initializes this file-based stack.
444 * @param $fileName File name of this stack
445 * @param $type Type of this stack (e.g. url_source for URL sources)
448 protected function initFileStack ($fileName, $type) {
449 // Get a file i/o pointer instance for stack file
450 $pointerInstance = ObjectFactory::createObjectByConfiguredName('file_raw_input_output_class', array($fileName));
452 // Get iterator instance
453 $iteratorInstance = ObjectFactory::createObjectByConfiguredName('file_io_iterator_class', array($pointerInstance));
455 // Is the instance implementing the right interface?
456 assert($iteratorInstance instanceof SeekableWritableFileIterator);
459 $this->setIteratorInstance($iteratorInstance);
461 // Is the file's header initialized?
462 if (!$this->isFileHeaderInitialized()) {
463 // No, then create it (which may pre-allocate the stack)
464 $this->createFileHeader();
466 // And pre-allocate a bit
467 $this->preAllocateFile();
470 // Load the file header
471 $this->readFileHeader();
473 // Count all entries in file
474 $this->analyzeStackFile();
477 * Get stack index instance. This can be used for faster
478 * "defragmentation" and startup.
480 $indexInstance = FileStackIndexFactory::createFileStackIndexInstance($fileName, $type);
483 $this->setIndexInstance($indexInstance);
487 * Adds a value to given stack
489 * @param $stackerName Name of the stack
490 * @param $value Value to add to this stacker
492 * @throws FullStackerException If the stack is full
494 protected function addValue ($stackerName, $value) {
496 if ($this->isStackFull($stackerName)) {
498 throw new FullStackerException(array($this, $stackerName, $value), self::EXCEPTION_STACKER_IS_FULL);
501 // Now add the value to the stack
502 $this->partialStub('stackerName=' . $stackerName . ',value[]=' . gettype($value));
506 * Get last value from named stacker
508 * @param $stackerName Name of the stack
509 * @return $value Value of last added value
510 * @throws EmptyStackerException If the stack is empty
512 protected function getLastValue ($stackerName) {
513 // Is the stack not yet initialized or full?
514 if ($this->isStackEmpty($stackerName)) {
515 // Throw an exception
516 throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY);
519 // Now get the last value
520 $this->partialStub('stackerName=' . $stackerName);
528 * Get first value from named stacker
530 * @param $stackerName Name of the stack
531 * @return $value Value of last added value
532 * @throws EmptyStackerException If the stack is empty
534 protected function getFirstValue ($stackerName) {
535 // Is the stack not yet initialized or full?
536 if ($this->isStackEmpty($stackerName)) {
537 // Throw an exception
538 throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY);
541 // Now get the first value
542 $this->partialStub('stackerName=' . $stackerName);
550 * "Pops" last entry from stack
552 * @param $stackerName Name of the stack
553 * @return $value Value "poped" from array
554 * @throws EmptyStackerException If the stack is empty
556 protected function popLast ($stackerName) {
557 // Is the stack not yet initialized or full?
558 if ($this->isStackEmpty($stackerName)) {
559 // Throw an exception
560 throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY);
563 // Now, remove the last entry, we don't care about the return value here, see elseif() block above
564 $this->partialStub('stackerName=' . $stackerName);
569 * "Pops" first entry from stack
571 * @param $stackerName Name of the stack
572 * @return $value Value "shifted" from array
573 * @throws EmptyStackerException If the named stacker is empty
575 protected function popFirst ($stackerName) {
576 // Is the stack not yet initialized or full?
577 if ($this->isStackEmpty($stackerName)) {
578 // Throw an exception
579 throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY);
582 // Now, remove the last entry, we don't care about the return value here, see elseif() block above
583 $this->partialStub('stackerName=' . $stackerName);
588 * Checks whether the given stack is full
590 * @param $stackerName Name of the stack
591 * @return $isFull Whether the stack is full
593 protected function isStackFull ($stackerName) {
594 // File-based stacks will only run full if the disk space is low.
595 // @TODO Please implement this, returning FALSE
603 * Checks whether the given stack is empty
605 * @param $stackerName Name of the stack
606 * @return $isEmpty Whether the stack is empty
607 * @throws NoStackerException If given stack is missing
609 public function isStackEmpty ($stackerName) {
610 // So, is the stack empty?
611 $isEmpty = (($this->getStackCount($stackerName)) == 0);
618 * Initializes given stacker
620 * @param $stackerName Name of the stack
621 * @param $forceReInit Force re-initialization
623 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
625 public function initStack ($stackerName, $forceReInit = FALSE) {
626 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
630 * Initializes all stacks
633 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
635 public function initStacks (array $stacks, $forceReInit = FALSE) {
636 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
640 * Checks whether the given stack is initialized (set in array $stackers)
642 * @param $stackerName Name of the stack
643 * @return $isInitialized Whether the stack is initialized
644 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
646 public function isStackInitialized ($stackerName) {
647 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
651 * Getter for size of given stack (array count)
653 * @param $stackerName Name of the stack
654 * @return $count Size of stack (array count)
656 public function getStackCount ($stackerName) {
657 // Now, simply return the found count value, this must be up-to-date then!
658 return $this->getCounter();