3 * A PackageFragmenter class to fragment package data into smaller chunks for
4 * delivery. This class calculates a final hash on the raw input data and
5 * fragments the data into smaller chunks after it has been encoded by a
6 * "outgoing encoding stream".
8 * All chunks are extended with a hash and a serial number to make it later
9 * easier to verify them and put them back in the right order and to, if
10 * required, request a re-delivery of an invalid chunk (e.g. hash didn't match).
11 * Also an "end-of-package" marker is being added as the last chunk to mark the
12 * end of of the whole package submission.
14 * @author Roland Haeder <webmaster@ship-simu.org>
16 * @copyright Copyright (c) 2007, 2008 Roland Haeder, 2009 - 2011 Hub Developer Team
17 * @license GNU GPL 3.0 or any newer version
18 * @link http://www.ship-simu.org
20 * This program is free software: you can redistribute it and/or modify
21 * it under the terms of the GNU General Public License as published by
22 * the Free Software Foundation, either version 3 of the License, or
23 * (at your option) any later version.
25 * This program is distributed in the hope that it will be useful,
26 * but WITHOUT ANY WARRANTY; without even the implied warranty of
27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28 * GNU General Public License for more details.
30 * You should have received a copy of the GNU General Public License
31 * along with this program. If not, see <http://www.gnu.org/licenses/>.
33 class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Registerable {
35 * Cached chunk size in bits
37 private $chunkSize = 0;
42 private $chunks = array();
45 * Array for chunk hashes
47 private $chunkHashes = array();
50 * Array for chunk pointers
52 private $chunkPointers = array();
55 * Array for processed packages
57 private $processedPackages = array();
62 private $serialNumber = 0x00000000;
65 * Maximum possible serial number
67 private $maxSerialNumber = 0;
70 * Length of largest possible serial number
72 const MAX_SERIAL_LENGTH = 8;
75 * Separator between chunk data, serial number and chunk hash
77 const CHUNK_DATA_HASH_SEPARATOR = '@';
80 * SEPARATOR for all chunk hashes
82 const CHUNK_HASH_SEPARATOR = ';';
85 * SEPARATOR between two chunks
87 const CHUNK_SEPARATOR = '|';
90 * Identifier for hash chunk
92 const HASH_CHUNK_IDENTIFIER = 'HASH-CHUNK:';
95 * Identifier for end-of-package marker
97 const END_OF_PACKAGE_IDENTIFIER = 'EOP:';
100 * Protected constructor
104 protected function __construct () {
105 // Call parent constructor
106 parent::__construct(__CLASS__);
108 // Init this fragmenter
109 $this->initFragmenter();
113 * Creates an instance of this class
115 * @return $fragmenterInstance An instance of a Fragmentable class
117 public static final function createPackageFragmenter () {
119 $fragmenterInstance = new PackageFragmenter();
121 // And also a crypto instance (for our encrypted messages)
122 $cryptoInstance = ObjectFactory::createObjectByConfiguredName('crypto_class');
123 $fragmenterInstance->setCryptoInstance($cryptoInstance);
125 // Return the prepared instance
126 return $fragmenterInstance;
130 * Initializes this fragmenter
134 private function initFragmenter () {
135 // Load some configuration entries and "cache" them:
136 // - Chunk size in bits
137 $this->chunkSize = $this->getConfigInstance()->getConfigEntry('package_chunk_size');
139 // - Maximum serial number
140 $this->maxSerialNumber = $this->hex2dec(str_repeat('f', self::MAX_SERIAL_LENGTH));
144 * Initializes the pointer for given final hash
146 * @param $finalHash Final hash to initialize pointer for
149 private function initPointer ($finalHash) {
150 $this->chunkPointers[$finalHash] = 0;
154 * "Getter" for processedPackages array index
156 * @param $packageData Raw package data array
157 * @return $index Array index for processedPackages
159 private function getProcessedPackagesIndex (array $packageData) {
161 $packageData[NetworkPackage::PACKAGE_DATA_SENDER] . NetworkPackage::PACKAGE_DATA_SEPARATOR .
162 $packageData[NetworkPackage::PACKAGE_DATA_RECIPIENT] . NetworkPackage::PACKAGE_DATA_SEPARATOR .
163 $packageData[NetworkPackage::PACKAGE_DATA_CONTENT] . NetworkPackage::PACKAGE_DATA_SEPARATOR
168 * Checks wether the given package data is already processed by this fragmenter
170 * @param $packageData Raw package data array
171 * @return $isProcessed Wether the package has been fragmented
173 private function isPackageProcessed (array $packageData) {
175 $index = $this->getProcessedPackagesIndex($packageData);
177 // Is the array index there?
179 (isset($this->processedPackages[$index]))
181 ($this->processedPackages[$index] === true)
189 * Marks the given package data as processed by this fragmenter
191 * @param $packageData Raw package data array
194 private function markPackageDataProcessed (array $packageData) {
195 // Remember it (until we may remove it)
196 $this->processedPackages[$this->getProcessedPackagesIndex($packageData)] = true;
200 * Getter for final hash from given package data
202 * @param $packageData Raw package data array
203 * @return $finalHash Final hash for package data
205 private function getFinalHashFromPackageData (array $packageData) {
206 // Make sure it is there
207 assert(isset($this->processedPackages[$this->getProcessedPackagesIndex($packageData)]));
210 return $this->processedPackages[$this->getProcessedPackagesIndex($packageData)];
214 * Get next chunk pointer for given final hash
216 * @param $finalHash Final hash to get current pointer for
218 private function getCurrentChunkPointer ($finalHash) {
219 // Is the final hash valid?
220 assert(strlen($finalHash) > 0);
222 // Is the pointer already initialized?
223 //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: finalHash=' . $finalHash);
224 assert(isset($this->chunkPointers[$finalHash]));
227 return $this->chunkPointers[$finalHash];
231 * Advance the chunk pointer for given final hash
233 * @param $finalHash Final hash to advance the pointer for
235 private function nextChunkPointer ($finalHash) {
236 // Is the pointer already initialized?
237 assert(isset($this->chunkPointers[$finalHash]));
240 $this->chunkPointers[$finalHash]++;
244 * "Getter" for data chunk size of given hash.
246 * @param $hash Hash to substract it's length
247 * @return $dataChunkSize The chunk size
249 private function getDataChunkSizeFromHash ($hash) {
250 // Calculate real (data) chunk size
253 ($this->chunkSize / 8) -
256 // Length of sperators
257 (strlen(self::CHUNK_DATA_HASH_SEPARATOR) * 2) -
258 // Length of max serial number
259 self::MAX_SERIAL_LENGTH
262 // This should be larger than zero bytes
263 assert($dataChunkSize > 0);
266 return $dataChunkSize;
270 * Generates a hash from raw data
272 * @param $rawData Raw data bytes to hash
273 * @return $hash Hash from the raw data
274 * @todo Implement a way to send non-announcement packages with extra-salt
276 private function generateHashFromRawData ($rawData) {
278 * Get the crypto instance and hash the data with no extra salt because
279 * the other peer doesn't have *this* peer's salt.
281 $hash = $this->getCryptoInstance()->hashString($rawData, '', false);
288 * "Getter" for the next hexadecimal-encoded serial number
290 * @return $encodedSerialNumber The next hexadecimal-encoded serial number
292 private function getNextHexSerialNumber () {
293 // Assert on maximum serial number length
294 assert($this->serialNumber <= $this->maxSerialNumber);
296 // Encode the current serial number
297 $encodedSerialNumber = $this->dec2Hex($this->serialNumber, self::MAX_SERIAL_LENGTH);
300 $this->serialNumber++;
302 // Return the encoded serial number
303 return $encodedSerialNumber;
307 * Appends an end-of-package chunk to the chunk list for given chunk and
308 * final hash. As of 23-March-2012 the format of this chunk will be as any
309 * regular one to keep things easy (KISS) in ChunkHandler class.
311 * @param $chunkHash Last chunk's hash
312 * @param $finalHash Final hash for raw (unencoded) data
315 private function appendEndOfPackageChunk ($chunkHash, $finalHash) {
316 // Generate end-of-package marker
318 self::END_OF_PACKAGE_IDENTIFIER .
319 $finalHash . self::CHUNK_HASH_SEPARATOR .
322 // Add it as regular chunk
323 $this->addChunkData($finalHash, $chunkData);
327 * Splits the given encoded data into smaller chunks, the size of the final
328 * and the SEPARATOR is being subtracted from chunk size to fit it into a
329 * TCP package (512 bytes).
331 * @param $rawData Raw data string
332 * @param $finalHash Final hash from the raw data
335 private function splitEncodedDataIntoChunks ($rawData, $finalHash) {
336 // Make sure final hashes with at least 32 bytes can pass
337 assert(strlen($finalHash) >= 32);
339 // Calculate real (data) chunk size
340 $dataChunkSize = $this->getDataChunkSizeFromHash($finalHash);
341 //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: dataChunkSize=' . $dataChunkSize);
347 for ($idx = 0; $idx < strlen($rawData); $idx += $dataChunkSize) {
348 // Get the next chunk
349 $chunkData = substr($rawData, $idx, $dataChunkSize);
351 // Add the chunk to the propper array and do all the stuff there
352 $this->addChunkData($finalHash, $chunkData);
356 //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: Raw data of ' . strlen($rawData) . ' bytes has been fragmented into ' . count($this->chunks[$finalHash]) . ' chunk(s).');
358 // Add end-of-package chunk
359 $this->appendEndOfPackageChunk($chunkHash, $finalHash);
363 * Adds the given chunk (raw data) to the proper array and hashes it for
366 * @param $finalHash Final hash for faster processing
367 * @param $chunkData Raw chunk data
370 private function addChunkData ($finalHash, $chunkData) {
372 $rawDataHash = $this->getCryptoInstance()->hashString($chunkData, '', false);
374 // Prepend the hash to the chunk
376 $rawDataHash . self::CHUNK_DATA_HASH_SEPARATOR .
377 $this->getNextHexSerialNumber() . self::CHUNK_DATA_HASH_SEPARATOR .
378 $chunkData . self::CHUNK_SEPARATOR
381 // Make sure the chunk is not larger than a TCP package can hold
382 assert(strlen($rawData) <= NetworkPackage::TCP_PACKAGE_SIZE);
384 // Add it to the array
385 //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: Adding ' . strlen($rawData) . ' bytes of a chunk ...');
386 $this->chunks[$finalHash][] = $rawData;
387 $this->chunkHashes[$finalHash][] = $rawDataHash;
391 * Prepends a chunk (or more) with all hashes from all chunks + final chunk.
393 * @param $rawData Raw data string
394 * @param $finalHash Final hash from the raw data
397 private function prependHashChunk ($rawData, $finalHash) {
398 // "Implode" the whole array of hashes into one string
399 $rawData = self::HASH_CHUNK_IDENTIFIER . implode(self::CHUNK_HASH_SEPARATOR, $this->chunkHashes[$finalHash]);
401 // Also get a hash from it
402 $chunkHash = $this->generateHashFromRawData($rawData);
404 // Calulcate chunk size
405 $dataChunkSize = $this->getDataChunkSizeFromHash($chunkHash);
407 // Now array_unshift() it to the two chunk arrays
408 for ($idx = 0; $idx < strlen($rawData); $idx += $dataChunkSize) {
409 // Get the next chunk
410 $chunk = substr($rawData, $idx, $dataChunkSize);
412 // Hash it and remember it in seperate array
413 $chunkHash = $this->getCryptoInstance()->hashString($chunk, '', false);
414 array_unshift($this->chunkHashes[$finalHash], $chunkHash);
416 // Prepend the hash to the chunk
418 $chunkHash . self::CHUNK_DATA_HASH_SEPARATOR .
419 $this->getNextHexSerialNumber() . self::CHUNK_DATA_HASH_SEPARATOR .
420 $chunk . self::CHUNK_SEPARATOR
423 // Make sure the chunk is not larger than a TCP package can hold
424 assert(strlen($chunk) <= NetworkPackage::TCP_PACKAGE_SIZE);
426 // Add it to the array
427 //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: Adding ' . strlen($chunk) . ' bytes of a chunk.');
428 array_unshift($this->chunks[$finalHash], $chunk);
433 * This method does "implode" the given package data array into one long
434 * string, splits it into small chunks, adds a serial number and checksum
435 * to all chunks and prepends a chunk with all hashes only in it. It will
436 * return the final hash for faster processing of packages.
438 * @param $packageData Raw package data array
439 * @param $helperInstance An instance of a ConnectionHelper class
440 * @return $finalHash Final hash for faster processing
441 * @todo $helperInstance is unused
443 public function fragmentPackageArray (array $packageData, ConnectionHelper $helperInstance) {
444 // Is this package already fragmented?
445 if (!$this->isPackageProcessed($packageData)) {
446 // Remove package status, the recipient doesn't need this
447 unset($packageData[NetworkPackage::PACKAGE_DATA_STATUS]);
449 // First we need to "implode" the array
450 $rawData = implode(NetworkPackage::PACKAGE_DATA_SEPARATOR, $packageData);
452 // Generate the final hash from the raw data (not encoded!)
453 $finalHash = $this->generateHashFromRawData($rawData);
456 $this->processedPackages[$this->getProcessedPackagesIndex($packageData)] = $finalHash;
459 $this->initPointer($finalHash);
461 // Split the encoded data into smaller chunks
462 $this->splitEncodedDataIntoChunks($rawData, $finalHash);
464 // Prepend a chunk with all hashes together
465 $this->prependHashChunk($rawData, $finalHash);
467 // Mark the package as fragmented
468 $this->markPackageDataProcessed($packageData);
470 // Get the final hash from the package data
471 $finalHash = $this->getFinalHashFromPackageData($packageData);
475 //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: finalHash[' . gettype($finalHash) . ']=' . $finalHash);
480 * This method gets the next chunk from the internal FIFO which should be
481 * sent to the given recipient. It will return an associative array where
482 * the key is the chunk hash and value the raw chunk data.
484 * @param $finalHash Final hash for faster lookup
485 * @return $rawDataChunk Raw package data chunk
486 * @throws AssertionException If $finalHash was not 'true'
488 public function getNextRawDataChunk ($finalHash) {
490 // Get current chunk index
491 $current = $this->getCurrentChunkPointer($finalHash);
492 } catch (AssertionException $e) {
493 // This may happen when the final hash is true
494 if ($finalHash === true) {
495 // Set current to null
498 // Throw the exception
503 // If there is no entry left, return an empty array
504 if ((!isset($this->chunkHashes[$finalHash][$current])) || (!isset($this->chunks[$finalHash][$current]))) {
505 // No more entries found
509 // Generate the array
510 $rawDataChunk = array(
511 $this->chunkHashes[$finalHash][$current] => $this->chunks[$finalHash][$current]
514 // Count one index up
515 $this->nextChunkPointer($finalHash);
517 // Return the chunk array
518 return $rawDataChunk;