X-Git-Url: https://git.mxchange.org/?p=hub.git;a=blobdiff_plain;f=application%2Fhub%2Fmain%2Fpackage%2Ffragmenter%2Fclass_PackageFragmenter.php;h=a816686c8cb6521e805d25ab405dbfd2cfaa27f0;hp=1cf6db7ea45fbfe2e0fe52638c5eab910119d5dd;hb=73aff29b9bc78031853b0b8c0fe0a8e04f66ac29;hpb=d391ecd222ae868265735539d68a2c148711b5e3 diff --git a/application/hub/main/package/fragmenter/class_PackageFragmenter.php b/application/hub/main/package/fragmenter/class_PackageFragmenter.php index 1cf6db7ea..a816686c8 100644 --- a/application/hub/main/package/fragmenter/class_PackageFragmenter.php +++ b/application/hub/main/package/fragmenter/class_PackageFragmenter.php @@ -1,15 +1,21 @@ + * All chunks are extended with a hash and a serial number to make it later + * easier to verify them and put them back in the right order and to, if + * required, request a re-delivery of an invalid chunk (e.g. hash didn't match). + * Also an "end-of-package" marker is being added as the last chunk to mark the + * end of of the whole package submission. + * + * @author Roland Haeder * @version 0.0.0 - * @copyright Copyright (c) 2007, 2008 Roland Haeder, 2009 - 2011 Hub Developer Team + * @copyright Copyright (c) 2007, 2008 Roland Haeder, 2009 - 2015 Hub Developer Team * @license GNU GPL 3.0 or any newer version - * @link http://www.ship-simu.org + * @link http://www.shipsimu.org * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -24,7 +30,7 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Registerable { +class PackageFragmenter extends BaseHubSystem implements Fragmentable, Registerable { /** * Cached chunk size in bits */ @@ -51,34 +57,45 @@ class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Reg private $processedPackages = array(); /** - * Serial number + * Serial numbers (array key is final hash) + */ + private $serialNumber = array(); + + /** + * Maximum possible serial number, "cache" for speeding up things */ - private $serialNumber = 0x00000000; + private $maxSerialNumber = 0; /** * Length of largest possible serial number */ - private $maxSerialLength = 8; + const MAX_SERIAL_LENGTH = 8; /** - * Maximum possible serial number + * Separator between chunk data, serial number and chunk hash */ - private $maxSerialNumber = 0; + const CHUNK_DATA_HASH_SEPARATOR = '@'; + /** - * Seperator between chunk data, serial number and chunk hash + * SEPARATOR for all chunk hashes */ - const CHUNK_DATA_HASH_SEPERATOR = '@'; + const CHUNK_HASH_SEPARATOR = ';'; /** - * Seperator for all chunk hashes + * SEPARATOR between two chunks */ - const CHUNK_HASH_SEPERATOR = ';'; + const CHUNK_SEPARATOR = '|'; /** * Identifier for hash chunk */ const HASH_CHUNK_IDENTIFIER = 'HASH-CHUNK:'; + /** + * Identifier for end-of-package marker + */ + const END_OF_PACKAGE_IDENTIFIER = 'EOP:'; + /** * Protected constructor * @@ -101,13 +118,7 @@ class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Reg // Get new instance $fragmenterInstance = new PackageFragmenter(); - // Get an output stream for all out-going packages - $streamInstance = ObjectFactory::createObjectByConfiguredName('node_raw_package_output_stream'); - - // And set it in this fragmenter - $fragmenterInstance->setOutputStreamInstance($streamInstance); - - // And also a crypto instance (for our encrypted messages) + // Get a crypto instance and set it here $cryptoInstance = ObjectFactory::createObjectByConfiguredName('crypto_class'); $fragmenterInstance->setCryptoInstance($cryptoInstance); @@ -126,7 +137,7 @@ class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Reg $this->chunkSize = $this->getConfigInstance()->getConfigEntry('package_chunk_size'); // - Maximum serial number - $this->maxSerialNumber = $this->hex2dec(str_repeat('f', $this->maxSerialLength)); + $this->maxSerialNumber = $this->hex2dec(str_repeat('f', self::MAX_SERIAL_LENGTH)); } /** @@ -147,17 +158,17 @@ class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Reg */ private function getProcessedPackagesIndex (array $packageData) { return ( - $packageData['sender'] . NetworkPackage::PACKAGE_DATA_SEPERATOR . - $packageData['recipient'] . NetworkPackage::PACKAGE_DATA_SEPERATOR . - $packageData['content'] . NetworkPackage::PACKAGE_DATA_SEPERATOR + $packageData[NetworkPackage::PACKAGE_DATA_SENDER] . NetworkPackage::PACKAGE_DATA_SEPARATOR . + $packageData[NetworkPackage::PACKAGE_DATA_RECIPIENT] . NetworkPackage::PACKAGE_DATA_SEPARATOR . + $packageData[NetworkPackage::PACKAGE_DATA_CONTENT] . NetworkPackage::PACKAGE_DATA_SEPARATOR ); } /** - * Checks wether the given package data is already processed by this fragmenter + * Checks whether the given package data is already processed by this fragmenter * * @param $packageData Raw package data array - * @return $isProcessed Wether the package has been fragmented + * @return $isProcessed Whether the package has been fragmented */ private function isPackageProcessed (array $packageData) { // Get array index @@ -165,8 +176,9 @@ class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Reg // Is the array index there? $isProcessed = ( - (isset($this->processedPackages[$index])) && - ($this->processedPackages[$index] === true) + (isset($this->processedPackages[$index])) + && + ($this->processedPackages[$index] === TRUE) ); // Return it @@ -181,7 +193,7 @@ class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Reg */ private function markPackageDataProcessed (array $packageData) { // Remember it (until we may remove it) - $this->processedPackages[$this->getProcessedPackagesIndex($packageData)] = true; + $this->processedPackages[$this->getProcessedPackagesIndex($packageData)] = TRUE; } /** @@ -208,7 +220,7 @@ class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Reg assert(strlen($finalHash) > 0); // Is the pointer already initialized? - //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: finalHash=' . $finalHash); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('PACKAGE-FRAGMENTER[' . __METHOD__ . ':' . __LINE__ . ']: finalHash[' . gettype($finalHash) . ']=' . $finalHash); assert(isset($this->chunkPointers[$finalHash])); // Return it @@ -225,6 +237,7 @@ class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Reg assert(isset($this->chunkPointers[$finalHash])); // Count one up + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('PACKAGE-FRAGMENTER[' . __METHOD__ . ':' . __LINE__ . ']: finalHash[' . gettype($finalHash) . ']=' . $finalHash); $this->chunkPointers[$finalHash]++; } @@ -242,9 +255,9 @@ class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Reg // Hash size strlen($hash) - // Length of sperators - (strlen(self::CHUNK_DATA_HASH_SEPERATOR) * 2) - + (strlen(self::CHUNK_DATA_HASH_SEPARATOR) * 2) - // Length of max serial number - $this->maxSerialLength + self::MAX_SERIAL_LENGTH ); // This should be larger than zero bytes @@ -259,114 +272,161 @@ class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Reg * * @param $rawData Raw data bytes to hash * @return $hash Hash from the raw data + * @todo Implement a way to send non-announcement packages with extra-salt */ private function generateHashFromRawData ($rawData) { - // Get the crypto instance and hash the data - $hash = $this->getCryptoInstance()->hashString($rawData); + /* + * Get the crypto instance and hash the data with no extra salt because + * the other peer doesn't have *this* peer's salt. + */ + $hash = $this->getCryptoInstance()->hashString($rawData, '', FALSE); // Return it return $hash; } /** - * "Getter" for the next hexadecimal-encoded serial number + * Appends an end-of-package chunk to the chunk list for given chunk and + * final hash. As of 23-March-2012 the format of this chunk will be as any + * regular one to keep things easy (KISS) in ChunkHandler class. * - * @return $encodedSerialNumber The next hexadecimal-encoded serial number + * @param $lastChunk Last chunk raw data + * @param $finalHash Final hash for raw (unencoded) data + * @return void */ - private function getNextHexSerialNumber () { - // Assert on maximum serial number length - assert($this->serialNumber <= $this->maxSerialNumber); - - // Encode the current serial number - $encodedSerialNumber = $this->dec2Hex($this->serialNumber, $this->maxSerialLength); - - // Count one up - $this->serialNumber++; - - // Return the encoded serial number - return $encodedSerialNumber; + private function appendEndOfPackageChunk ($lastChunk, $finalHash) { + // Generate end-of-package marker + $chunkData = + self::END_OF_PACKAGE_IDENTIFIER . + $finalHash . self::CHUNK_HASH_SEPARATOR . + $this->generateHashFromRawData($lastChunk); + + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('PACKAGE-FRAGMENTER[' . __METHOD__ . ':' . __LINE__ . ']: Adding EOP chunk with size of ' . strlen($chunkData) . ',finalHash=' . $finalHash . ' ...'); + + // Add it as regular chunk + $this->addChunkData($finalHash, $chunkData); } /** * Splits the given encoded data into smaller chunks, the size of the final - * and the seperator is being subtracted from chunk size to fit it into a + * and the SEPARATOR is being subtracted from chunk size to fit it into a * TCP package (512 bytes). * - * @param $encodedData Encoded data string - * @param $finalHash Final hash from the raw data + * @param $rawData Raw data string + * @param $finalHash Final hash from the raw data * @return void */ - private function splitEncodedDataIntoChunks ($encodedData, $finalHash) { + private function splitEncodedDataIntoChunks ($rawData, $finalHash) { // Make sure final hashes with at least 32 bytes can pass assert(strlen($finalHash) >= 32); // Calculate real (data) chunk size $dataChunkSize = $this->getDataChunkSizeFromHash($finalHash); - //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: dataChunkSize=' . $dataChunkSize); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('PACKAGE-FRAGMENTER[' . __METHOD__ . ':' . __LINE__ . ']: dataChunkSize=' . $dataChunkSize); + + // Init variables + $chunkHash = ''; + $chunkData = ''; // Now split it up - for ($idx = 0; $idx < strlen($encodedData); $idx += $dataChunkSize) { + for ($idx = 0; $idx < strlen($rawData); $idx += $dataChunkSize) { // Get the next chunk - $chunk = substr($encodedData, $idx, $dataChunkSize); + $chunkData = substr($rawData, $idx, $dataChunkSize); - // Hash it and remember it in seperate array - $chunkHash = $this->getCryptoInstance()->hashString($chunk); - $this->chunkHashes[$finalHash][] = $chunkHash; - - // Prepend the hash to the chunk - $chunk = $chunkHash . self::CHUNK_DATA_HASH_SEPERATOR . $this->getNextHexSerialNumber() . self::CHUNK_DATA_HASH_SEPERATOR . $chunk; - - // Make sure the chunk is not larger than a TCP package can hold - assert(strlen($chunk) <= NetworkPackage::TCP_PACKAGE_SIZE); - - // Add it to the array - //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: Adding ' . strlen($chunk) . ' bytes of a chunk.'); - $this->chunks[$finalHash][] = $chunk; + // Add the chunk to the propper array and do all the stuff there + $this->addChunkData($finalHash, $chunkData); } // END - for // Debug output - $this->debugOutput('FRAGMENTER: Encoded data of ' . strlen($encodedData) . ' bytes has been fragmented into ' . count($this->chunks[$finalHash]) . ' chunk(s).'); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('PACKAGE-FRAGMENTER[' . __METHOD__ . ':' . __LINE__ . ']: Raw data of ' . strlen($rawData) . ' bytes has been fragmented into ' . count($this->chunks[$finalHash]) . ' chunk(s).'); + + // Add end-of-package chunk + $this->appendEndOfPackageChunk($chunkData, $finalHash); } /** - * Prepends a chunk (or more) with all hashes from all chunks + final chunk. + * Adds the given chunk (raw data) to the proper array and hashes it for + * later verfication. * - * @param $encodedData Encoded data string - * @param $finalHash Final hash from the raw data + * @param $finalHash Final hash for faster processing + * @param $chunkData Raw chunk data + * @param $prepend Whether append (default) or prepend the chunk * @return void */ - private function prependHashChunk ($encodedData, $finalHash) { - // "Implode" the whole array of hashes into one string - $rawData = self::HASH_CHUNK_IDENTIFIER . implode(self::CHUNK_HASH_SEPERATOR, $this->chunkHashes[$finalHash]); + private function addChunkData ($finalHash, $chunkData, $prepend = FALSE) { + // Hash it + $rawDataHash = $this->getCryptoInstance()->hashString($chunkData, '', FALSE); + + // Prepend the hash to the chunk + $rawData = ( + $rawDataHash . self::CHUNK_DATA_HASH_SEPARATOR . + $this->getNextHexSerialNumber($finalHash) . self::CHUNK_DATA_HASH_SEPARATOR . + $chunkData . self::CHUNK_SEPARATOR + ); - // Also get a hash from it - $chunkHash = $this->generateHashFromRawData($rawData); + // Make sure the chunk is not larger than a TCP package can hold + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('PACKAGE-FRAGMENTER[' . __METHOD__ . ':' . __LINE__ . ']: assert: ' . strlen($rawData) . '/' . NetworkPackage::TCP_PACKAGE_SIZE . ' ...'); + // @TODO This assert broke packages where the hash chunk was very large: assert(strlen($rawData) <= NetworkPackage::TCP_PACKAGE_SIZE); - // Also encode this one - $encodedData = $this->getOutputStreamInstance()->streamData($rawData); + // Add it to the array + if ($prepend === TRUE) { + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('PACKAGE-FRAGMENTER[' . __METHOD__ . ':' . __LINE__ . ']: Prepending ' . strlen($rawData) . ' bytes of a chunk, finalHash=' . $finalHash . ' ...'); + array_unshift($this->chunkHashes[$finalHash], $rawDataHash); + array_unshift($this->chunks[$finalHash] , $rawData); + } else { + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('PACKAGE-FRAGMENTER[' . __METHOD__ . ':' . __LINE__ . ']: Appending ' . strlen($rawData) . ' bytes of a chunk, finalHash=' . $finalHash . ' ...'); + + // Is the array there? + if (!isset($this->chunks[$finalHash])) { + // Then initialize it + $this->chunks[$finalHash] = array(); + $this->chunkHashes[$finalHash] = array(); + } // END - if + + // Add both + array_push($this->chunks[$finalHash] , $rawData); + array_push($this->chunkHashes[$finalHash], $rawDataHash); + } + } - // Calulcate chunk size - $dataChunkSize = $this->getDataChunkSizeFromHash($chunkHash); + /** + * Prepends a chunk (or more) with all hashes from all chunks + final chunk. + * + * @param $finalHash Final hash from the raw data + * @return void + */ + private function prependHashChunk ($finalHash) { + // "Implode" the whole array of hashes into one string + $rawData = self::HASH_CHUNK_IDENTIFIER . implode(self::CHUNK_HASH_SEPARATOR, $this->chunkHashes[$finalHash]); - // Now array_unshift() it to the two chunk arrays - for ($idx = 0; $idx < strlen($encodedData); $idx += $dataChunkSize) { - // Get the next chunk - $chunk = substr($encodedData, $idx, $dataChunkSize); + // Prepend chunk + $this->addChunkData($finalHash, $rawData, TRUE); + } - // Hash it and remember it in seperate array - $chunkHash = $this->getCryptoInstance()->hashString($chunk); - array_unshift($this->chunkHashes[$finalHash], $chunkHash); + /** + * "Getter" for the next hexadecimal-encoded serial number + * + * @param $finalHash Final hash + * @return $encodedSerialNumber The next hexadecimal-encoded serial number + */ + public function getNextHexSerialNumber ($finalHash) { + // Assert on maximum serial number length + assert(isset($this->serialNumber[$finalHash])); + assert($this->serialNumber[$finalHash] <= $this->maxSerialNumber); - // Prepend the hash to the chunk - $chunk = $chunkHash . self::CHUNK_DATA_HASH_SEPERATOR . $this->getNextHexSerialNumber() . self::CHUNK_DATA_HASH_SEPERATOR . $chunk; + // Encode the current serial number + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('PACKAGE-FRAGMENTER[' . __METHOD__ . ':' . __LINE__ . ']: serialNumber[' . $finalHash . ']=' . $this->serialNumber[$finalHash]); + $encodedSerialNumber = $this->dec2Hex($this->serialNumber[$finalHash], self::MAX_SERIAL_LENGTH); - // Make sure the chunk is not larger than a TCP package can hold - assert(strlen($chunk) <= NetworkPackage::TCP_PACKAGE_SIZE); + // Count one up + $this->serialNumber[$finalHash]++; - // Add it to the array - //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: Adding ' . strlen($chunk) . ' bytes of a chunk.'); - array_unshift($this->chunks[$finalHash], $chunk); - } // END - for + // Return the encoded serial number + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('PACKAGE-FRAGMENTER[' . __METHOD__ . ':' . __LINE__ . ']: encodedSerialNumber=' . $encodedSerialNumber); + return $encodedSerialNumber; } /** @@ -375,16 +435,16 @@ class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Reg * to all chunks and prepends a chunk with all hashes only in it. It will * return the final hash for faster processing of packages. * - * @param $packageData Raw package data array - * @param $connectionInstance A helper instance for connections - * @return $finalHash Final hash for faster processing - * @todo $connectionInstance is unused + * @param $packageData Raw package data array + * @param $helperInstance An instance of a ConnectionHelper class + * @return $finalHash Final hash for faster processing + * @todo $helperInstance is unused */ - public function fragmentPackageArray (array $packageData, BaseConnectionHelper $connectionInstance) { + public function fragmentPackageArray (array $packageData, ConnectionHelper $helperInstance) { // Is this package already fragmented? if (!$this->isPackageProcessed($packageData)) { // First we need to "implode" the array - $rawData = implode(NetworkPackage::PACKAGE_DATA_SEPERATOR, $packageData); + $rawData = implode(NetworkPackage::PACKAGE_DATA_SEPARATOR, $packageData); // Generate the final hash from the raw data (not encoded!) $finalHash = $this->generateHashFromRawData($rawData); @@ -392,17 +452,15 @@ class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Reg // Remember it $this->processedPackages[$this->getProcessedPackagesIndex($packageData)] = $finalHash; - // Init pointer + // Init pointer and reset serial number $this->initPointer($finalHash); - - // Encode the package for delivery - $encodedData = $this->getOutputStreamInstance()->streamData($rawData); + $this->resetSerialNumber($finalHash); // Split the encoded data into smaller chunks - $this->splitEncodedDataIntoChunks($encodedData, $finalHash); + $this->splitEncodedDataIntoChunks($rawData, $finalHash); // Prepend a chunk with all hashes together - $this->prependHashChunk($encodedData, $finalHash); + $this->prependHashChunk($finalHash); // Mark the package as fragmented $this->markPackageDataProcessed($packageData); @@ -412,7 +470,7 @@ class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Reg } // Return final hash - //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: finalHash[' . gettype($finalHash) . ']=' . $finalHash); + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('PACKAGE-FRAGMENTER[' . __METHOD__ . ':' . __LINE__ . ']: finalHash[' . gettype($finalHash) . ']=' . $finalHash); return $finalHash; } @@ -423,16 +481,20 @@ class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Reg * * @param $finalHash Final hash for faster lookup * @return $rawDataChunk Raw package data chunk + * @throws AssertionException If $finalHash was not 'TRUE' */ public function getNextRawDataChunk ($finalHash) { + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput('PACKAGE-FRAGMENTER[' . __METHOD__ . ':' . __LINE__ . ']: finalHash[' . gettype($finalHash) . ']=' . $finalHash); + try { // Get current chunk index $current = $this->getCurrentChunkPointer($finalHash); } catch (AssertionException $e) { - // This may happen when the final hash is true - if ($finalHash === true) { + // This may happen when the final hash is TRUE + if ($finalHash === TRUE) { // Set current to null - $current = null; + $current = NULL; } else { // Throw the exception throw $e; @@ -442,9 +504,13 @@ class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Reg // If there is no entry left, return an empty array if ((!isset($this->chunkHashes[$finalHash][$current])) || (!isset($this->chunks[$finalHash][$current]))) { // No more entries found + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(__METHOD__. ': finalHash=' . $finalHash . ',current=' . $current . ' - No more entries found!'); return array(); } // END - if + // Debug message + //* NOISY-DEBUG: */ self::createDebugInstance(__CLASS__)->debugOutput(__METHOD__. ': finalHash=' . $finalHash . ',current=' . $current . ',chunkHashes()=' . count($this->chunkHashes[$finalHash]) .' - Entry choosen ...'); + // Generate the array $rawDataChunk = array( $this->chunkHashes[$finalHash][$current] => $this->chunks[$finalHash][$current] @@ -456,6 +522,20 @@ class PackageFragmenter extends BaseFrameworkSystem implements Fragmentable, Reg // Return the chunk array return $rawDataChunk; } + + /** + * Resets the serial number to zero for given final hash + * + * @param $finalHash Final hash to reset counter for + * @return void + */ + public function resetSerialNumber ($finalHash) { + // Final hash must be set + assert((is_string($finalHash)) && (!empty($finalHash))); + + // Reset/set serial number + $this->serialNumber[$finalHash] = 0; + } } // [EOF]