3 * A PackageFragmenter class to fragment package data into smaller chunks for
4 * delivery. This class calculates a final hash on the raw input data and
5 * fragments the data into smaller chunks after it has been encoded by a
6 * "outgoing encoding stream".
8 * All chunks are extended with a hash and a serial number to make it later
9 * easier to verify them and put them back in the right order and to, if
10 * required, request a re-delivery of an invalid chunk (e.g. hash didn't match).
11 * Also an "end-of-package" marker is being added as the last chunk to mark the
12 * end of of the whole package submission.
14 * @author Roland Haeder <webmaster@ship-simu.org>
16 * @copyright Copyright (c) 2007, 2008 Roland Haeder, 2009 - 2011 Hub Developer Team
17 * @license GNU GPL 3.0 or any newer version
18 * @link http://www.ship-simu.org
20 * This program is free software: you can redistribute it and/or modify
21 * it under the terms of the GNU General Public License as published by
22 * the Free Software Foundation, either version 3 of the License, or
23 * (at your option) any later version.
25 * This program is distributed in the hope that it will be useful,
26 * but WITHOUT ANY WARRANTY; without even the implied warranty of
27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28 * GNU General Public License for more details.
30 * You should have received a copy of the GNU General Public License
31 * along with this program. If not, see <http://www.gnu.org/licenses/>.
33 class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Registerable {
35 * Cached chunk size in bits
37 private $chunkSize = 0;
42 private $chunks = array();
45 * Array for chunk hashes
47 private $chunkHashes = array();
50 * Array for chunk pointers
52 private $chunkPointers = array();
55 * Array for processed packages
57 private $processedPackages = array();
62 private $serialNumber = 0x00000000;
65 * Maximum possible serial number
67 private $maxSerialNumber = 0;
70 * Length of largest possible serial number
72 const MAX_SERIAL_LENGTH = 8;
75 * Separator between chunk data, serial number and chunk hash
77 const CHUNK_DATA_HASH_SEPARATOR = '@';
80 * SEPARATOR for all chunk hashes
82 const CHUNK_HASH_SEPARATOR = ';';
85 * SEPARATOR between two chunks
87 const CHUNK_SEPARATOR = '|';
90 * Identifier for hash chunk
92 const HASH_CHUNK_IDENTIFIER = 'HASH-CHUNK:';
95 * Identifier for end-of-package marker
97 const END_OF_PACKAGE_IDENTIFIER = 'EOP:';
100 * Protected constructor
104 protected function __construct () {
105 // Call parent constructor
106 parent::__construct(__CLASS__);
108 // Init this fragmenter
109 $this->initFragmenter();
113 * Creates an instance of this class
115 * @return $fragmenterInstance An instance of a Fragmentable class
117 public static final function createPackageFragmenter () {
119 $fragmenterInstance = new PackageFragmenter();
121 // And also a crypto instance (for our encrypted messages)
122 $cryptoInstance = ObjectFactory::createObjectByConfiguredName('crypto_class');
123 $fragmenterInstance->setCryptoInstance($cryptoInstance);
125 // Return the prepared instance
126 return $fragmenterInstance;
130 * Initializes this fragmenter
134 private function initFragmenter () {
135 // Load some configuration entries and "cache" them:
136 // - Chunk size in bits
137 $this->chunkSize = $this->getConfigInstance()->getConfigEntry('package_chunk_size');
139 // - Maximum serial number
140 $this->maxSerialNumber = $this->hex2dec(str_repeat('f', self::MAX_SERIAL_LENGTH));
144 * Initializes the pointer for given final hash
146 * @param $finalHash Final hash to initialize pointer for
149 private function initPointer ($finalHash) {
150 $this->chunkPointers[$finalHash] = 0;
154 * "Getter" for processedPackages array index
156 * @param $packageData Raw package data array
157 * @return $index Array index for processedPackages
159 private function getProcessedPackagesIndex (array $packageData) {
161 $packageData[NetworkPackage::PACKAGE_DATA_SENDER] . NetworkPackage::PACKAGE_DATA_SEPARATOR .
162 $packageData[NetworkPackage::PACKAGE_DATA_RECIPIENT] . NetworkPackage::PACKAGE_DATA_SEPARATOR .
163 $packageData[NetworkPackage::PACKAGE_DATA_CONTENT] . NetworkPackage::PACKAGE_DATA_SEPARATOR
168 * Checks wether the given package data is already processed by this fragmenter
170 * @param $packageData Raw package data array
171 * @return $isProcessed Wether the package has been fragmented
173 private function isPackageProcessed (array $packageData) {
175 $index = $this->getProcessedPackagesIndex($packageData);
177 // Is the array index there?
179 (isset($this->processedPackages[$index]))
181 ($this->processedPackages[$index] === true)
189 * Marks the given package data as processed by this fragmenter
191 * @param $packageData Raw package data array
194 private function markPackageDataProcessed (array $packageData) {
195 // Remember it (until we may remove it)
196 $this->processedPackages[$this->getProcessedPackagesIndex($packageData)] = true;
200 * Getter for final hash from given package data
202 * @param $packageData Raw package data array
203 * @return $finalHash Final hash for package data
205 private function getFinalHashFromPackageData (array $packageData) {
206 // Make sure it is there
207 assert(isset($this->processedPackages[$this->getProcessedPackagesIndex($packageData)]));
210 return $this->processedPackages[$this->getProcessedPackagesIndex($packageData)];
214 * Get next chunk pointer for given final hash
216 * @param $finalHash Final hash to get current pointer for
218 private function getCurrentChunkPointer ($finalHash) {
219 // Is the final hash valid?
220 assert(strlen($finalHash) > 0);
222 // Is the pointer already initialized?
223 //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: finalHash=' . $finalHash);
224 assert(isset($this->chunkPointers[$finalHash]));
227 return $this->chunkPointers[$finalHash];
231 * Advance the chunk pointer for given final hash
233 * @param $finalHash Final hash to advance the pointer for
235 private function nextChunkPointer ($finalHash) {
236 // Is the pointer already initialized?
237 assert(isset($this->chunkPointers[$finalHash]));
240 $this->chunkPointers[$finalHash]++;
244 * "Getter" for data chunk size of given hash.
246 * @param $hash Hash to substract it's length
247 * @return $dataChunkSize The chunk size
249 private function getDataChunkSizeFromHash ($hash) {
250 // Calculate real (data) chunk size
253 ($this->chunkSize / 8) -
256 // Length of sperators
257 (strlen(self::CHUNK_DATA_HASH_SEPARATOR) * 2) -
258 // Length of max serial number
259 self::MAX_SERIAL_LENGTH
262 // This should be larger than zero bytes
263 assert($dataChunkSize > 0);
266 return $dataChunkSize;
270 * Generates a hash from raw data
272 * @param $rawData Raw data bytes to hash
273 * @return $hash Hash from the raw data
275 private function generateHashFromRawData ($rawData) {
277 * Get the crypto instance and hash the data with no extra salt because
278 * the other peer doesn't have *this* peer's salt.
280 $hash = $this->getCryptoInstance()->hashString($rawData, '', false);
287 * "Getter" for the next hexadecimal-encoded serial number
289 * @return $encodedSerialNumber The next hexadecimal-encoded serial number
291 private function getNextHexSerialNumber () {
292 // Assert on maximum serial number length
293 assert($this->serialNumber <= $this->maxSerialNumber);
295 // Encode the current serial number
296 $encodedSerialNumber = $this->dec2Hex($this->serialNumber, self::MAX_SERIAL_LENGTH);
299 $this->serialNumber++;
301 // Return the encoded serial number
302 return $encodedSerialNumber;
306 * Appends an end-of-package chunk to the chunk list for given chunk and
309 * @param $chunkHash Last chunk's hash
310 * @param $finalHash Final hash for raw (unencoded) data
313 private function appendEndOfPackageChunk ($chunkHash, $finalHash) {
314 // Generate end-of-package marker
316 self::END_OF_PACKAGE_IDENTIFIER .
317 $finalHash . self::CHUNK_HASH_SEPARATOR .
318 $chunkHash . self::CHUNK_SEPARATOR;
320 // Also get a hash from it
321 $eopHash = $this->generateHashFromRawData($rawData);
323 // Append it to the eop's data and hash array
324 $this->chunkHashes[$finalHash][] = $eopHash;
325 $this->chunks[$finalHash][] = $rawData;
329 * Splits the given encoded data into smaller chunks, the size of the final
330 * and the SEPARATOR is being subtracted from chunk size to fit it into a
331 * TCP package (512 bytes).
333 * @param $rawData Raw data string
334 * @param $finalHash Final hash from the raw data
337 private function splitEncodedDataIntoChunks ($rawData, $finalHash) {
338 // Make sure final hashes with at least 32 bytes can pass
339 assert(strlen($finalHash) >= 32);
341 // Calculate real (data) chunk size
342 $dataChunkSize = $this->getDataChunkSizeFromHash($finalHash);
343 //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: dataChunkSize=' . $dataChunkSize);
349 for ($idx = 0; $idx < strlen($rawData); $idx += $dataChunkSize) {
350 // Get the next chunk
351 $chunk = substr($rawData, $idx, $dataChunkSize);
353 // Hash it and remember it in seperate array
354 $chunkHash = $this->getCryptoInstance()->hashString($chunk, '', false);
355 $this->chunkHashes[$finalHash][] = $chunkHash;
357 // Prepend the hash to the chunk
359 $chunkHash . self::CHUNK_DATA_HASH_SEPARATOR .
360 $this->getNextHexSerialNumber() . self::CHUNK_DATA_HASH_SEPARATOR .
361 $chunk . self::CHUNK_SEPARATOR
364 // Make sure the chunk is not larger than a TCP package can hold
365 assert(strlen($chunk) <= NetworkPackage::TCP_PACKAGE_SIZE);
367 // Add it to the array
368 //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: Adding ' . strlen($chunk) . ' bytes of a chunk.');
369 $this->chunks[$finalHash][] = $chunk;
373 //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: Raw data of ' . strlen($rawData) . ' bytes has been fragmented into ' . count($this->chunks[$finalHash]) . ' chunk(s).');
375 // Add end-of-package chunk
376 $this->appendEndOfPackageChunk($chunkHash, $finalHash);
380 * Prepends a chunk (or more) with all hashes from all chunks + final chunk.
382 * @param $rawData Raw data string
383 * @param $finalHash Final hash from the raw data
386 private function prependHashChunk ($rawData, $finalHash) {
387 // "Implode" the whole array of hashes into one string
388 $rawData = self::HASH_CHUNK_IDENTIFIER . implode(self::CHUNK_HASH_SEPARATOR, $this->chunkHashes[$finalHash]);
390 // Also get a hash from it
391 $chunkHash = $this->generateHashFromRawData($rawData);
393 // Calulcate chunk size
394 $dataChunkSize = $this->getDataChunkSizeFromHash($chunkHash);
396 // Now array_unshift() it to the two chunk arrays
397 for ($idx = 0; $idx < strlen($rawData); $idx += $dataChunkSize) {
398 // Get the next chunk
399 $chunk = substr($rawData, $idx, $dataChunkSize);
401 // Hash it and remember it in seperate array
402 $chunkHash = $this->getCryptoInstance()->hashString($chunk, '', false);
403 array_unshift($this->chunkHashes[$finalHash], $chunkHash);
405 // Prepend the hash to the chunk
407 $chunkHash . self::CHUNK_DATA_HASH_SEPARATOR .
408 $this->getNextHexSerialNumber() . self::CHUNK_DATA_HASH_SEPARATOR .
409 $chunk . self::CHUNK_SEPARATOR
412 // Make sure the chunk is not larger than a TCP package can hold
413 assert(strlen($chunk) <= NetworkPackage::TCP_PACKAGE_SIZE);
415 // Add it to the array
416 //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: Adding ' . strlen($chunk) . ' bytes of a chunk.');
417 array_unshift($this->chunks[$finalHash], $chunk);
422 * This method does "implode" the given package data array into one long
423 * string, splits it into small chunks, adds a serial number and checksum
424 * to all chunks and prepends a chunk with all hashes only in it. It will
425 * return the final hash for faster processing of packages.
427 * @param $packageData Raw package data array
428 * @param $helperInstance An instance of a ConnectionHelper class
429 * @return $finalHash Final hash for faster processing
430 * @todo $helperInstance is unused
432 public function fragmentPackageArray (array $packageData, ConnectionHelper $helperInstance) {
433 // Is this package already fragmented?
434 if (!$this->isPackageProcessed($packageData)) {
435 // Remove package status, the recipient doesn't need this
436 unset($packageData[NetworkPackage::PACKAGE_DATA_STATUS]);
438 // First we need to "implode" the array
439 $rawData = implode(NetworkPackage::PACKAGE_DATA_SEPARATOR, $packageData);
441 // Generate the final hash from the raw data (not encoded!)
442 $finalHash = $this->generateHashFromRawData($rawData);
445 $this->processedPackages[$this->getProcessedPackagesIndex($packageData)] = $finalHash;
448 $this->initPointer($finalHash);
450 // Split the encoded data into smaller chunks
451 $this->splitEncodedDataIntoChunks($rawData, $finalHash);
453 // Prepend a chunk with all hashes together
454 $this->prependHashChunk($rawData, $finalHash);
456 // Mark the package as fragmented
457 $this->markPackageDataProcessed($packageData);
459 // Get the final hash from the package data
460 $finalHash = $this->getFinalHashFromPackageData($packageData);
464 //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: finalHash[' . gettype($finalHash) . ']=' . $finalHash);
469 * This method gets the next chunk from the internal FIFO which should be
470 * sent to the given recipient. It will return an associative array where
471 * the key is the chunk hash and value the raw chunk data.
473 * @param $finalHash Final hash for faster lookup
474 * @return $rawDataChunk Raw package data chunk
475 * @throws AssertionException If $finalHash was not 'true'
477 public function getNextRawDataChunk ($finalHash) {
479 // Get current chunk index
480 $current = $this->getCurrentChunkPointer($finalHash);
481 } catch (AssertionException $e) {
482 // This may happen when the final hash is true
483 if ($finalHash === true) {
484 // Set current to null
487 // Throw the exception
492 // If there is no entry left, return an empty array
493 if ((!isset($this->chunkHashes[$finalHash][$current])) || (!isset($this->chunks[$finalHash][$current]))) {
494 // No more entries found
498 // Generate the array
499 $rawDataChunk = array(
500 $this->chunkHashes[$finalHash][$current] => $this->chunks[$finalHash][$current]
503 // Count one index up
504 $this->nextChunkPointer($finalHash);
506 // Return the chunk array
507 return $rawDataChunk;