3 namespace Org\Mxchange\CoreFramework\Stacker\Filesystem;
5 // Import framework stuff
6 use Org\Mxchange\CoreFramework\Factory\ObjectFactory;
7 use Org\Mxchange\CoreFramework\Filesystem\File\BaseBinaryFile;
8 use Org\Mxchange\CoreFramework\Generic\UnsupportedOperationException;
9 use Org\Mxchange\CoreFramework\Iterator\Filesystem\SeekableWritableFileIterator;
10 use Org\Mxchange\CoreFramework\Stacker\BaseStacker;
13 * A general file-based stack class
15 * @author Roland Haeder <webmaster@ship-simu.org>
17 * @copyright Copyright (c) 2007, 2008 Roland Haeder, 2009 - 2017 Core Developer Team
18 * @license GNU GPL 3.0 or any newer version
19 * @link http://www.ship-simu.org
21 * This program is free software: you can redistribute it and/or modify
22 * it under the terms of the GNU General Public License as published by
23 * the Free Software Foundation, either version 3 of the License, or
24 * (at your option) any later version.
26 * This program is distributed in the hope that it will be useful,
27 * but WITHOUT ANY WARRANTY; without even the implied warranty of
28 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
29 * GNU General Public License for more details.
31 * You should have received a copy of the GNU General Public License
32 * along with this program. If not, see <http://www.gnu.org/licenses/>.
34 abstract class BaseFileStack extends BaseStacker {
36 * Magic for this stack
38 const STACK_MAGIC = 'STACKv0.1';
41 * Name of array index for gap position
43 const ARRAY_INDEX_GAP_POSITION = 'gap';
46 * Name of array index for hash
48 const ARRAY_INDEX_HASH = 'hash';
51 * Name of array index for length of raw data
53 const ARRAY_INDEX_DATA_LENGTH = 'length';
56 * Protected constructor
58 * @param $className Name of the class
61 protected function __construct ($className) {
62 // Call parent constructor
63 parent::__construct($className);
67 * Reads the file header
70 * @todo To hard assertions here, better rewrite them to exceptions
72 public function readFileHeader () {
73 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
75 // First rewind to beginning as the header sits at the beginning ...
76 $this->getIteratorInstance()->rewind();
78 // Then read it (see constructor for calculation)
79 $data = $this->getIteratorInstance()->read($this->getIteratorInstance()->getHeaderSize());
80 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] Read %d bytes (%d wanted).', __METHOD__, __LINE__, strlen($data), $this->getIteratorInstance()->getHeaderSize()));
82 // Have all requested bytes been read?
83 assert(strlen($data) == $this->getIteratorInstance()->getHeaderSize());
84 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
86 // Last character must be the separator
87 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] data(-1)=%s', __METHOD__, __LINE__, dechex(ord(substr($data, -1, 1)))));
88 assert(substr($data, -1, 1) == chr(BaseBinaryFile::SEPARATOR_HEADER_ENTRIES));
89 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
91 // Okay, then remove it
92 $data = substr($data, 0, -1);
94 // And update seek position
95 $this->getIteratorInstance()->updateSeekPosition();
102 * 2 => current seek position
104 $header = explode(chr(BaseBinaryFile::SEPARATOR_HEADER_DATA), $data);
107 $this->getIteratorInstance()->setHeader($header);
109 // Check if the array has only 3 elements
110 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] header(%d)=%s', __METHOD__, __LINE__, count($header), print_r($header, true)));
111 assert(count($header) == 3);
112 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
115 assert($header[0] == self::STACK_MAGIC);
116 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
118 // Check length of count and seek position
119 assert(strlen($header[1]) == BaseBinaryFile::LENGTH_COUNT);
120 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
121 assert(strlen($header[2]) == BaseBinaryFile::LENGTH_POSITION);
122 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] Passed assert().', __METHOD__, __LINE__));
124 // Decode count and seek position
125 $header[1] = hex2bin($header[1]);
126 $header[2] = hex2bin($header[2]);
128 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
132 * Flushes the file header
136 public function flushFileHeader () {
137 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] CALLED!', __METHOD__, __LINE__));
139 // Put all informations together
140 $header = sprintf('%s%s%s%s%s%s',
144 // Separator magic<->count
145 chr(BaseBinaryFile::SEPARATOR_HEADER_DATA),
147 // Total entries (will be zero) and pad it to 20 chars
148 str_pad($this->dec2hex($this->getIteratorInstance()->getCounter()), BaseBinaryFile::LENGTH_COUNT, '0', STR_PAD_LEFT),
150 // Separator count<->seek position
151 chr(BaseBinaryFile::SEPARATOR_HEADER_DATA),
153 // Position (will be zero)
154 str_pad($this->dec2hex($this->getIteratorInstance()->getSeekPosition(), 2), BaseBinaryFile::LENGTH_POSITION, '0', STR_PAD_LEFT),
156 // Separator position<->entries
157 chr(BaseBinaryFile::SEPARATOR_HEADER_ENTRIES)
160 // Write it to disk (header is always at seek position 0)
161 $this->getIteratorInstance()->writeData(0, $header, false);
163 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] EXIT!', __METHOD__, __LINE__));
167 * Initializes this file-based stack.
169 * @param $fileName File name of this stack
170 * @param $type Type of this stack (e.g. url_source for URL sources)
172 * @todo Currently the stack file is not cached, please implement a memory-handling class and if enough RAM is found, cache the whole stack file.
174 protected function initFileStack ($fileName, $type) {
175 // Get a stack file instance
176 $fileInstance = ObjectFactory::createObjectByConfiguredName('stack_file_class', array($fileName, $this));
178 // Get iterator instance
179 $iteratorInstance = ObjectFactory::createObjectByConfiguredName('file_iterator_class', array($fileInstance));
181 // Is the instance implementing the right interface?
182 assert($iteratorInstance instanceof SeekableWritableFileIterator);
185 $this->setIteratorInstance($iteratorInstance);
187 // Calculate header size
188 $this->getIteratorInstance()->setHeaderSize(
189 strlen(self::STACK_MAGIC) +
190 strlen(chr(BaseBinaryFile::SEPARATOR_HEADER_DATA)) +
191 BaseBinaryFile::LENGTH_COUNT +
192 strlen(chr(BaseBinaryFile::SEPARATOR_HEADER_DATA)) +
193 BaseBinaryFile::LENGTH_POSITION +
194 strlen(chr(BaseBinaryFile::SEPARATOR_HEADER_ENTRIES))
197 // Init counters and gaps array
198 $this->getIteratorInstance()->initCountersGapsArray();
200 // Is the file's header initialized?
201 if (!$this->getIteratorInstance()->isFileHeaderInitialized()) {
202 // No, then create it (which may pre-allocate the stack)
203 $this->getIteratorInstance()->createFileHeader();
205 // And pre-allocate a bit
206 $this->getIteratorInstance()->preAllocateFile('file_stack');
209 // Load the file header
210 $this->readFileHeader();
212 // Count all entries in file
213 $this->getIteratorInstance()->analyzeFile();
216 * Get stack index instance. This can be used for faster
217 * "defragmentation" and startup.
219 $indexInstance = FileStackIndexFactory::createFileStackIndexInstance($fileName, $type);
222 $this->setIndexInstance($indexInstance);
226 * Adds a value to given stack
228 * @param $stackerName Name of the stack
229 * @param $value Value to add to this stacker
231 * @throws FullStackerException If the stack is full
233 protected function addValue ($stackerName, $value) {
235 if ($this->isStackFull($stackerName)) {
237 throw new FullStackerException(array($this, $stackerName, $value), self::EXCEPTION_STACKER_IS_FULL);
241 //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput('[' . __METHOD__ . ':' . __LINE__ . '] stackerName=' . $stackerName . ',value[' . gettype($value) . ']=' . print_r($value, true));
243 // No objects/resources are allowed as their serialization takes to long
244 assert(!is_object($value));
245 assert(!is_resource($value));
248 * Now add the value to the file stack which returns gap position, a
249 * hash and length of the raw data.
251 $data = $this->getIteratorInstance()->writeValueToFile($stackerName, $value);
253 // Add the hash and gap position to the index
254 $this->getIndexInstance()->addHashToIndex($stackerName, $data);
258 * Get last value from named stacker
260 * @param $stackerName Name of the stack
261 * @return $value Value of last added value
262 * @throws EmptyStackerException If the stack is empty
264 protected function getLastValue ($stackerName) {
265 // Is the stack not yet initialized or full?
266 if ($this->isStackEmpty($stackerName)) {
267 // Throw an exception
268 throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY);
271 // Now get the last value
272 /* NOISY-DEBUG: */ $this->partialStub('[' . __METHOD__ . ':' . __LINE__ . '] stackerName=' . $stackerName);
280 * Get first value from named stacker
282 * @param $stackerName Name of the stack
283 * @return $value Value of last added value
284 * @throws EmptyStackerException If the stack is empty
286 protected function getFirstValue ($stackerName) {
287 // Is the stack not yet initialized or full?
288 if ($this->isStackEmpty($stackerName)) {
289 // Throw an exception
290 throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY);
293 // Now get the first value
294 /* NOISY-DEBUG: */ $this->partialStub('[' . __METHOD__ . ':' . __LINE__ . '] stackerName=' . $stackerName);
302 * "Pops" last entry from stack
304 * @param $stackerName Name of the stack
305 * @return $value Value "poped" from array
306 * @throws EmptyStackerException If the stack is empty
308 protected function popLast ($stackerName) {
309 // Is the stack not yet initialized or full?
310 if ($this->isStackEmpty($stackerName)) {
311 // Throw an exception
312 throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY);
315 // Now, remove the last entry, we don't care about the return value here, see elseif() block above
316 /* NOISY-DEBUG: */ $this->partialStub('[' . __METHOD__ . ':' . __LINE__ . '] stackerName=' . $stackerName);
321 * "Pops" first entry from stack
323 * @param $stackerName Name of the stack
324 * @return $value Value "shifted" from array
325 * @throws EmptyStackerException If the named stacker is empty
327 protected function popFirst ($stackerName) {
328 // Is the stack not yet initialized or full?
329 if ($this->isStackEmpty($stackerName)) {
330 // Throw an exception
331 throw new EmptyStackerException(array($this, $stackerName), self::EXCEPTION_STACKER_IS_EMPTY);
334 // Now, remove the last entry, we don't care about the return value here, see elseif() block above
335 /* NOISY-DEBUG: */ $this->partialStub('[' . __METHOD__ . ':' . __LINE__ . '] stackerName=' . $stackerName);
340 * Checks whether the given stack is full
342 * @param $stackerName Name of the stack
343 * @return $isFull Whether the stack is full
345 protected function isStackFull ($stackerName) {
346 // File-based stacks will only run full if the disk space is low.
347 // @TODO Please implement this, returning false
355 * Checks whether the given stack is empty
357 * @param $stackerName Name of the stack
358 * @return $isEmpty Whether the stack is empty
359 * @throws NoStackerException If given stack is missing
361 public function isStackEmpty ($stackerName) {
362 // So, is the stack empty?
363 $isEmpty = (($this->getStackCount($stackerName)) == 0);
370 * Initializes given stacker
372 * @param $stackerName Name of the stack
373 * @param $forceReInit Force re-initialization
375 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
377 public function initStack ($stackerName, $forceReInit = false) {
378 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
382 * Initializes all stacks
385 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
387 public function initStacks (array $stacks, $forceReInit = false) {
388 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
392 * Checks whether the given stack is initialized (set in array $stackers)
394 * @param $stackerName Name of the stack
395 * @return $isInitialized Whether the stack is initialized
396 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
398 public function isStackInitialized ($stackerName) {
399 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
403 * Determines whether the EOF has been reached
405 * @return $isEndOfFileReached Whether the EOF has been reached
406 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
408 public function isEndOfFileReached () {
409 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
413 * Getter for size of given stack (array count)
415 * @param $stackerName Name of the stack
416 * @return $count Size of stack (array count)
418 public function getStackCount ($stackerName) {
419 // Now, simply return the found count value, this must be up-to-date then!
420 return $this->getIteratorInstance()->getCounter();
424 * Calculates minimum length for one entry/block
426 * @return $length Minimum length for one entry/block
428 public function calculateMinimumBlockLength () {
431 // Length of entry group
432 BaseBinaryFile::LENGTH_GROUP + strlen(chr(BaseBinaryFile::SEPARATOR_GROUP_HASH)) +
434 self::getHashLength() + strlen(chr(BaseBinaryFile::SEPARATOR_HASH_VALUE)) + 1 +
436 strlen(chr(BaseBinaryFile::SEPARATOR_ENTRIES));
443 * Initializes counter for valid entries, arrays for damaged entries and
444 * an array for gap seek positions. If you call this method on your own,
445 * please re-analyze the file structure. So you are better to call
446 * analyzeFile() instead of this method.
449 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
451 public function initCountersGapsArray () {
452 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
456 * Getter for header size
458 * @return $totalEntries Size of file header
459 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
461 public final function getHeaderSize () {
462 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
466 * Setter for header size
468 * @param $headerSize Size of file header
470 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
472 public final function setHeaderSize ($headerSize) {
473 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
477 * Getter for header array
479 * @return $totalEntries Size of file header
480 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
482 public final function getHeader () {
483 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
489 * @param $header Array for a file header
491 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
493 public final function setHeader (array $header) {
494 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
498 * Updates seekPosition attribute from file to avoid to much access on file.
501 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
503 public function updateSeekPosition () {
504 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
508 * Getter for total entries
510 * @return $totalEntries Total entries in this file
511 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
513 public final function getCounter () {
514 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
518 * Writes data at given position
520 * @param $seekPosition Seek position
521 * @param $data Data to be written
522 * @param $flushHeader Whether to flush the header (default: flush)
524 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
526 public function writeData ($seekPosition, $data, $flushHeader = true) {
527 /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] seekPosition=%s,data[]=%s,flushHeader=%d', __METHOD__, __LINE__, $seekPosition, gettype($data), intval($flushHeader)));
528 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
532 * Writes given value to the file and returns a hash and gap position for it
534 * @param $groupId Group identifier
535 * @param $value Value to be added to the stack
536 * @return $data Hash and gap position
537 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
539 public function writeValueToFile ($groupId, $value) {
540 /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] groupId=%s,value[%s]=%s', __METHOD__, __LINE__, $groupId, gettype($value), print_r($value, true)));
541 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
545 * Searches for next suitable gap the given length of data can fit in
546 * including padding bytes.
548 * @param $length Length of raw data
549 * @return $seekPosition Found next gap's seek position
550 * @throws UnsupportedOperationException This method is not (and maybe never will be) supported
552 public function searchNextGap ($length) {
553 /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] length=%s', __METHOD__, __LINE__, $length));
554 throw new UnsupportedOperationException(array($this, __FUNCTION__, $this->getIteratorInstance()->getPointerInstance()), self::EXCEPTION_UNSPPORTED_OPERATION);
558 * "Getter" for file size
560 * @return $fileSize Size of currently loaded file
562 public function getFileSize () {
563 // Call iterator's method
564 return $this->getIteratorInstance()->getFileSize();
568 * Writes given raw data to the file and returns a gap position and length
570 * @param $groupId Group identifier
571 * @param $hash Hash from encoded value
572 * @param $encoded Encoded value to be written to the file
573 * @return $data Gap position and length of the raw data
575 public function writeDataToFreeGap ($groupId, $hash, $encoded) {
577 /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] groupId=%s,hash=%s,encoded()=%d - CALLED!', __METHOD__, __LINE__, $groupId, $hash, strlen($encoded)));
579 // Raw data been written to the file
580 $rawData = sprintf('%s%s%s%s%s',
582 BaseBinaryFile::SEPARATOR_GROUP_HASH,
584 BaseBinaryFile::SEPARATOR_HASH_VALUE,
589 /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] groupId=%s,hash=%s,rawData()=%d', __METHOD__, __LINE__, $groupId, $hash, strlen($rawData)));
591 // Search for next free gap
592 $gapPosition = $this->getIteratorInstance()->searchNextGap(strlen($rawData));
594 // Gap position cannot be smaller than header length + 1
595 assert($gapPosition > $this->getIteratorInstance()->getHeaderSize());
598 /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] groupId=%s,hash=%s,gapPosition=%s', __METHOD__, __LINE__, $groupId, $hash, $gapPosition));
600 // Then write the data at that gap
601 $this->getIteratorInstance()->writeData($gapPosition, $rawData);
604 /* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__, __LINE__)->debugOutput(sprintf('[%s:%d:] groupId=%s,hash=%s,rawData()=%d - EXIT!', __METHOD__, __LINE__, $groupId, $hash, strlen($rawData)));
606 // Return gap position, hash and length of raw data
608 self::ARRAY_INDEX_GAP_POSITION => $gapPosition,
609 self::ARRAY_INDEX_HASH => $hash,
610 self::ARRAY_INDEX_DATA_LENGTH => strlen($rawData)