From 6124e1c12e45fd628f93e3bf27ed481952494146 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Fri, 27 Apr 2012 19:36:27 +0000 Subject: [PATCH] 'hub' project continued: - prependHashChunk() does now add the full hash-chunk to the array, not chunked by itself as before. This change made reading the package much easier. - Added more debug lines - TODOs.txt updated --- .../connection/class_BaseConnectionHelper.php | 37 ++++++---- .../fragmenter/class_PackageFragmenter.php | 72 +++++++++---------- docs/TODOs.txt | 8 +-- 3 files changed, 61 insertions(+), 56 deletions(-) diff --git a/application/hub/main/helper/connection/class_BaseConnectionHelper.php b/application/hub/main/helper/connection/class_BaseConnectionHelper.php index 685d079e3..c85464642 100644 --- a/application/hub/main/helper/connection/class_BaseConnectionHelper.php +++ b/application/hub/main/helper/connection/class_BaseConnectionHelper.php @@ -91,6 +91,12 @@ class BaseConnectionHelper extends BaseHubHelper implements Registerable, Protoc // Register this connection helper Registry::getRegistry()->addInstance('connection', $this); + + // Get the fragmenter instance + $fragmenterInstance = FragmenterFactory::createFragmenterInstance('package'); + + // Set it here + $this->setFragmenterInstance($fragmenterInstance); } /** @@ -295,11 +301,8 @@ class BaseConnectionHelper extends BaseHubHelper implements Registerable, Protoc * @return $chunkData Raw data chunk */ private function getRawDataFromPackageArray (array $packageData) { - // Get the fragmenter instance - $fragmenterInstance = FragmenterFactory::createFragmenterInstance('package'); - // Implode the package data array and fragement the resulting string, returns the final hash - $finalHash = $fragmenterInstance->fragmentPackageArray($packageData, $this); + $finalHash = $this->getFragmenterInstance()->fragmentPackageArray($packageData, $this); if ($finalHash !== true) { $this->currentFinalHash = $finalHash; } // END - if @@ -308,7 +311,7 @@ class BaseConnectionHelper extends BaseHubHelper implements Registerable, Protoc //* NOISY-DEBUG: */ $this->debugOutput('CONNECTION: currentFinalHash=' . $this->currentFinalHash); // Get the next raw data chunk from the fragmenter - $rawDataChunk = $fragmenterInstance->getNextRawDataChunk($this->currentFinalHash); + $rawDataChunk = $this->getFragmenterInstance()->getNextRawDataChunk($this->currentFinalHash); // Get chunk hashes and chunk data $chunkHashes = array_keys($rawDataChunk); @@ -316,14 +319,17 @@ class BaseConnectionHelper extends BaseHubHelper implements Registerable, Protoc // Is the required data there? //* NOISY-DEBUG: */ $this->debugOutput('CONNECTION: chunkHashes[]=' . count($chunkHashes) . ',chunkData[]=' . count($chunkData)); + //* NOISY-DEBUG: */ $this->debugOutput('chunkData='.print_r($chunkData,true)); if ((isset($chunkHashes[0])) && (isset($chunkData[0]))) { // Remember this chunk as queued $this->queuedChunks[$chunkHashes[0]] = $chunkData[0]; // Return the raw data + //* NOISY-DEBUG: */ $this->debugOutput('CONNECTION: Returning ' . strlen($chunkData[0]) . ' bytes from ' . __METHOD__ . ' ...'); return $chunkData[0]; } else { // Return zero string + //* NOISY-DEBUG: */ $this->debugOutput('CONNECTION: Returning zero bytes from ' . __METHOD__ . '!'); return ''; } } @@ -359,12 +365,13 @@ class BaseConnectionHelper extends BaseHubHelper implements Registerable, Protoc $totalSentBytes = 0; // Fill sending buffer with data - while ((strlen($rawData) < $bufferSize) && (strlen($dataStream) > 0)) { + while (strlen($dataStream) > 0) { // Convert the package data array to a raw data stream $dataStream = $this->getRawDataFromPackageArray($packageData); //* NOISY-DEBUG: */ $this->debugOutput('CONNECTION: Adding ' . strlen($dataStream) . ' bytes to the sending buffer ...'); $rawData .= $dataStream; } // END - while + //* NOISY-DEBUG: */ $this->debugOutput('CONNECTION: rawData[' . strlen($rawData) . ']=' . $rawData); // Nothing to sent is bad news, so assert on it assert(strlen($rawData) > 0); @@ -385,7 +392,13 @@ class BaseConnectionHelper extends BaseHubHelper implements Registerable, Protoc while ($sentBytes !== false) { // And deliver it //* NOISY-DEBUG: */ $this->debugOutput('CONNECTION: Sending out ' . strlen($encodedData) . ' bytes,bufferSize=' . $bufferSize . ',diff=' . $this->diff); - $sentBytes = @socket_write($socketResource, $encodedData, ($bufferSize - $this->diff)); + if ($this->diff >= 0) { + // Send all out (encodedData is smaller than or equal buffer size) + $sentBytes = @socket_write($socketResource, $encodedData, ($bufferSize - $this->diff)); + } else { + // Send buffer size out + $sentBytes = @socket_write($socketResource, $encodedData, $bufferSize); + } // If there was an error, we don't continue here if ($sentBytes === false) { @@ -396,7 +409,7 @@ class BaseConnectionHelper extends BaseHubHelper implements Registerable, Protoc throw new InvalidSocketException(array($this, $socketResource, $socketError, $errorMessage), BaseListener::EXCEPTION_INVALID_SOCKET); } elseif (($sentBytes == 0) && (strlen($encodedData) > 0)) { // Nothing sent means we are done - //* NOISY-DEBUG: */ $this->debugOutput('CONNECTION: All sent! (' . __LINE__ . ')'); + //* NOISY-DEBUG: */ $this->debugOutput('CONNECTION: All sent! (LINE=' . __LINE__ . ')'); break; } @@ -416,13 +429,13 @@ class BaseConnectionHelper extends BaseHubHelper implements Registerable, Protoc // Can we abort? if (strlen($encodedData) <= 0) { // Abort here, all sent! - //* NOISY-DEBUG: */ $this->debugOutput('CONNECTION: All sent! (' . __LINE__ . ')'); + //* NOISY-DEBUG: */ $this->debugOutput('CONNECTION: All sent! (LINE=' . __LINE__ . ')'); break; } // END - if } // END - while // Return sent bytes - //* NOISY-DEBUG: */ $this->debugOutput('CONNECTION: totalSentBytes=' . $totalSentBytes); + //* NOISY-DEBUG: */ $this->debugOutput('CONNECTION: totalSentBytes=' . $totalSentBytes . ',diff=' . $this->diff); return $totalSentBytes; } @@ -432,7 +445,7 @@ class BaseConnectionHelper extends BaseHubHelper implements Registerable, Protoc * @return void */ protected final function markConnectionShuttedDown () { - /* NOISY-DEBUG: */ $this->debugOutput('CONNECTION: ' . $this->__toString() . ' has been marked as shutted down'); + //* NOISY-DEBUG: */ $this->debugOutput('CONNECTION: ' . $this->__toString() . ' has been marked as shutted down'); $this->shuttedDown = true; // And remove the (now invalid) socket @@ -445,7 +458,7 @@ class BaseConnectionHelper extends BaseHubHelper implements Registerable, Protoc * @return $shuttedDown Whether this connection is shutted down */ public final function isShuttedDown () { - /* NOISY-DEBUG: */ $this->debugOutput('CONNECTION: ' . $this->__toString() . ',shuttedDown=' . intval($this->shuttedDown)); + //* NOISY-DEBUG: */ $this->debugOutput('CONNECTION: ' . $this->__toString() . ',shuttedDown=' . intval($this->shuttedDown)); return $this->shuttedDown; } diff --git a/application/hub/main/package/fragmenter/class_PackageFragmenter.php b/application/hub/main/package/fragmenter/class_PackageFragmenter.php index d1c59c7fb..3431846a6 100644 --- a/application/hub/main/package/fragmenter/class_PackageFragmenter.php +++ b/application/hub/main/package/fragmenter/class_PackageFragmenter.php @@ -220,7 +220,7 @@ class PackageFragmenter extends BaseHubSystem implements Fragmentable, Registera assert(strlen($finalHash) > 0); // Is the pointer already initialized? - //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: finalHash=' . $finalHash); + //* NOISY-DEBUG: */ $this->debugOutput(__METHOD__ . ': finalHash[' . gettype($finalHash) . ']=' . $finalHash); assert(isset($this->chunkPointers[$finalHash])); // Return it @@ -237,6 +237,7 @@ class PackageFragmenter extends BaseHubSystem implements Fragmentable, Registera assert(isset($this->chunkPointers[$finalHash])); // Count one up + //* NOISY-DEBUG: */ $this->debugOutput(__METHOD__ . ': finalHash[' . gettype($finalHash) . ']=' . $finalHash); $this->chunkPointers[$finalHash]++; } @@ -300,6 +301,9 @@ class PackageFragmenter extends BaseHubSystem implements Fragmentable, Registera $finalHash . self::CHUNK_HASH_SEPARATOR . $this->generateHashFromRawData($lastChunk); + // Debug message + //* NOISY-DEBUG: */ $this->debugOutput(__METHOD__ . ': Adding EOP chunk with size of ' . strlen($chunkData) . ',finalHash=' . $finalHash . ' ...'); + // Add it as regular chunk $this->addChunkData($finalHash, $chunkData); } @@ -319,7 +323,7 @@ class PackageFragmenter extends BaseHubSystem implements Fragmentable, Registera // Calculate real (data) chunk size $dataChunkSize = $this->getDataChunkSizeFromHash($finalHash); - //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: dataChunkSize=' . $dataChunkSize); + //* NOISY-DEBUG: */ $this->debugOutput(__METHOD__ . ': dataChunkSize=' . $dataChunkSize); // Init variables $chunkHash = ''; @@ -335,7 +339,7 @@ class PackageFragmenter extends BaseHubSystem implements Fragmentable, Registera } // END - for // Debug output - //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: Raw data of ' . strlen($rawData) . ' bytes has been fragmented into ' . count($this->chunks[$finalHash]) . ' chunk(s).'); + //* NOISY-DEBUG: */ $this->debugOutput(__METHOD__ . ': Raw data of ' . strlen($rawData) . ' bytes has been fragmented into ' . count($this->chunks[$finalHash]) . ' chunk(s).'); // Add end-of-package chunk $this->appendEndOfPackageChunk($chunkData, $finalHash); @@ -347,9 +351,10 @@ class PackageFragmenter extends BaseHubSystem implements Fragmentable, Registera * * @param $finalHash Final hash for faster processing * @param $chunkData Raw chunk data + * @param $prepend Whether append (default) or prepend the chunk * @return void */ - private function addChunkData ($finalHash, $chunkData) { + private function addChunkData ($finalHash, $chunkData, $prepend = false) { // Hash it $rawDataHash = $this->getCryptoInstance()->hashString($chunkData, '', false); @@ -364,51 +369,31 @@ class PackageFragmenter extends BaseHubSystem implements Fragmentable, Registera assert(strlen($rawData) <= NetworkPackage::TCP_PACKAGE_SIZE); // Add it to the array - //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: Adding ' . strlen($rawData) . ' bytes of a chunk ...'); - $this->chunks[$finalHash][] = $rawData; - $this->chunkHashes[$finalHash][] = $rawDataHash; + if ($prepend === true) { + // Debug message + //* NOISY-DEBUG: */ $this->debugOutput(__METHOD__ . ': Prepending ' . strlen($rawData) . ' bytes of a chunk, finalHash=' . $finalHash . ' ...'); + array_unshift($this->chunkHashes[$finalHash], $rawDataHash); + array_unshift($this->chunks[$finalHash] , $rawData); + } else { + // Debug message + //* NOISY-DEBUG: */ $this->debugOutput(__METHOD__ . ': Appending ' . strlen($rawData) . ' bytes of a chunk, finalHash=' . $finalHash . ' ...'); + $this->chunks[$finalHash][] = $rawData; + $this->chunkHashes[$finalHash][] = $rawDataHash; + } } /** * Prepends a chunk (or more) with all hashes from all chunks + final chunk. * - * @param $rawData Raw data string * @param $finalHash Final hash from the raw data * @return void */ - private function prependHashChunk ($rawData, $finalHash) { + private function prependHashChunk ($finalHash) { // "Implode" the whole array of hashes into one string $rawData = self::HASH_CHUNK_IDENTIFIER . implode(self::CHUNK_HASH_SEPARATOR, $this->chunkHashes[$finalHash]); - // Also get a hash from it - $chunkHash = $this->generateHashFromRawData($rawData); - - // Calulcate chunk size - $dataChunkSize = $this->getDataChunkSizeFromHash($chunkHash); - - // Now array_unshift() it to the two chunk arrays - for ($idx = 0; $idx < strlen($rawData); $idx += $dataChunkSize) { - // Get the next chunk - $chunk = substr($rawData, $idx, $dataChunkSize); - - // Hash it and remember it in seperate array - $chunkHash = $this->getCryptoInstance()->hashString($chunk, '', false); - array_unshift($this->chunkHashes[$finalHash], $chunkHash); - - // Prepend the hash to the chunk - $chunk = - $chunkHash . self::CHUNK_DATA_HASH_SEPARATOR . - $this->getNextHexSerialNumber() . self::CHUNK_DATA_HASH_SEPARATOR . - $chunk . self::CHUNK_SEPARATOR - ; - - // Make sure the chunk is not larger than a TCP package can hold - assert(strlen($chunk) <= NetworkPackage::TCP_PACKAGE_SIZE); - - // Add it to the array - //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: Adding ' . strlen($chunk) . ' bytes of a chunk.'); - array_unshift($this->chunks[$finalHash], $chunk); - } // END - for + // Prepend chunk + $this->addChunkData($finalHash, $rawData, true); } /** @@ -463,7 +448,7 @@ class PackageFragmenter extends BaseHubSystem implements Fragmentable, Registera $this->splitEncodedDataIntoChunks($rawData, $finalHash); // Prepend a chunk with all hashes together - $this->prependHashChunk($rawData, $finalHash); + $this->prependHashChunk($finalHash); // Mark the package as fragmented $this->markPackageDataProcessed($packageData); @@ -473,7 +458,7 @@ class PackageFragmenter extends BaseHubSystem implements Fragmentable, Registera } // Return final hash - //* NOISY-DEBUG: */ $this->debugOutput('FRAGMENTER: finalHash[' . gettype($finalHash) . ']=' . $finalHash); + //* NOISY-DEBUG: */ $this->debugOutput(__METHOD__ . ': finalHash[' . gettype($finalHash) . ']=' . $finalHash); return $finalHash; } @@ -487,6 +472,9 @@ class PackageFragmenter extends BaseHubSystem implements Fragmentable, Registera * @throws AssertionException If $finalHash was not 'true' */ public function getNextRawDataChunk ($finalHash) { + // Debug message + //* NOISY-DEBUG: */ $this->debugOutput(__METHOD__ . ': finalHash[' . gettype($finalHash) . ']=' . $finalHash); + try { // Get current chunk index $current = $this->getCurrentChunkPointer($finalHash); @@ -504,9 +492,13 @@ class PackageFragmenter extends BaseHubSystem implements Fragmentable, Registera // If there is no entry left, return an empty array if ((!isset($this->chunkHashes[$finalHash][$current])) || (!isset($this->chunks[$finalHash][$current]))) { // No more entries found + $this->debugOutput(__METHOD__. ': finalHash=' . $finalHash . ',current=' . $current . ' - No more entries found!'); return array(); } // END - if + // Debug message + $this->debugOutput(__METHOD__. ': finalHash=' . $finalHash . ',current=' . $current . ',chunkHashes()=' . count($this->chunkHashes[$finalHash]) .' - Entry choosen ...'); + // Generate the array $rawDataChunk = array( $this->chunkHashes[$finalHash][$current] => $this->chunks[$finalHash][$current] diff --git a/docs/TODOs.txt b/docs/TODOs.txt index 34b5f2bbc..db2d967ce 100644 --- a/docs/TODOs.txt +++ b/docs/TODOs.txt @@ -47,8 +47,8 @@ ./application/hub/main/handler/network/class_BaseRawDataHandler.php:156: // @TODO Numeric or alpha-numeric index? ./application/hub/main/handler/network/udp/class_UdpRawDataHandler.php:58: * @todo 0% ./application/hub/main/handler/tasks/class_TaskHandler.php:138: // @TODO Messurement can be added around this call -./application/hub/main/helper/connection/class_BaseConnectionHelper.php:182: // @TODO Move this to the socket error handler -./application/hub/main/helper/connection/class_BaseConnectionHelper.php:210: * @todo Rewrite the while() loop to a iterator to not let the software stay very long here +./application/hub/main/helper/connection/class_BaseConnectionHelper.php:188: // @TODO Move this to the socket error handler +./application/hub/main/helper/connection/class_BaseConnectionHelper.php:216: * @todo Rewrite the while() loop to a iterator to not let the software stay very long here ./application/hub/main/helper/connection/tcp/class_TcpConnectionHelper.php:10: * @todo Find an interface for hub helper ./application/hub/main/helper/connection/tcp/class_TcpConnectionHelper.php:145: * @todo We may want to implement a filter for ease notification of other objects like our pool ./application/hub/main/helper/connection/tcp/class_TcpConnectionHelper.php:49: * @todo $errorCode/-Message are now in handleSocketError()'s call-back methods @@ -95,8 +95,8 @@ ./application/hub/main/package/class_NetworkPackage.php:688: // @TODO Add some logging here ./application/hub/main/package/class_NetworkPackage.php:787: // @TODO Add some content here ./application/hub/main/package/class_NetworkPackage.php:826: * @todo This may be enchanced for outgoing packages? -./application/hub/main/package/fragmenter/class_PackageFragmenter.php:274: * @todo Implement a way to send non-announcement packages with extra-salt -./application/hub/main/package/fragmenter/class_PackageFragmenter.php:442: * @todo $helperInstance is unused +./application/hub/main/package/fragmenter/class_PackageFragmenter.php:275: * @todo Implement a way to send non-announcement packages with extra-salt +./application/hub/main/package/fragmenter/class_PackageFragmenter.php:427: * @todo $helperInstance is unused ./application/hub/main/producer/cruncher/keys/class_CruncherKeyProducer.php:106: // @TODO Do something with it ./application/hub/main/producer/cruncher/keys/class_CruncherKeyProducer.php:62: * @todo Find something for init phase of this key producer ./application/hub/main/producer/cruncher/keys/class_CruncherKeyProducer.php:72: * @todo ~30% done -- 2.39.2