From 1c28686d33a76fa9e3e5bd90f737b20bfe211575 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Thu, 17 Feb 2011 11:12:15 +0000 Subject: [PATCH] Support for chunked HTTP messages added, some code encapsulated: - Support for 'Transfer-Encoding: chunked' HTTP header added (it might still be buggy, please report) - New function removeHttpHeaderFromResponse() introduced (which encapsulates some code dublicates) - TODOs.txt updated --- DOCS/TODOs.txt | 15 +- inc/functions.php | 295 ++++++++++++++++++++++++++--- inc/libs/yoomedia_functions.php | 16 +- inc/modules/admin/what-updates.php | 16 +- 4 files changed, 284 insertions(+), 58 deletions(-) diff --git a/DOCS/TODOs.txt b/DOCS/TODOs.txt index aa7d5a56f1..8d3c261673 100644 --- a/DOCS/TODOs.txt +++ b/DOCS/TODOs.txt @@ -44,13 +44,13 @@ ./inc/extensions-functions.php:434:// @TODO Change from ext_id to ext_name (not just even the variable! ;-) ) ./inc/extensions-functions.php:564: // @TODO Extension is loaded, what next? ./inc/functions.php:110: // @TODO Extension 'msg' does not exist -./inc/functions.php:1512: // @TODO Move this SQL code into a function, let's say 'getTimestampFromPoolId($id) ? -./inc/functions.php:1600: // @TODO Are these convertions still required? -./inc/functions.php:1618:// @TODO Rewrite this function to use readFromFile() and writeToFile() +./inc/functions.php:1611: // @TODO Move this SQL code into a function, let's say 'getTimestampFromPoolId($id) ? +./inc/functions.php:1699: // @TODO Are these convertions still required? +./inc/functions.php:1717:// @TODO Rewrite this function to use readFromFile() and writeToFile() ./inc/functions.php:181:// @TODO Rewrite this to an extension 'smtp' -./inc/functions.php:2200: // @TODO Find a way to cache this -./inc/functions.php:2301: // @TODO This is still very static, rewrite it somehow -./inc/functions.php:2481: // @TODO Rename column data_type to e.g. mail_status +./inc/functions.php:2299: // @TODO Find a way to cache this +./inc/functions.php:2400: // @TODO This is still very static, rewrite it somehow +./inc/functions.php:2580: // @TODO Rename column data_type to e.g. mail_status ./inc/gen_sql_patches.php:95:// @TODO Rewrite this to a filter ./inc/install-functions.php:57: // @TODO DEACTIVATED: changeDataInInclude(getCachePath() . 'config-local.php', 'OUTPUT-MODE', "setConfigEntry('OUTPUT_MODE', '", "');", postRequestParameter('omode'), 0); ./inc/language/de.php:1083: // @TODO Rewrite these two constants @@ -117,7 +117,8 @@ ./inc/modules/admin/what-adminedit.php:56: // @TODO Kill all constants in this file ./inc/modules/admin/what-admins_mails.php:59: // @TODO Can this be rewritten to an API function? ./inc/modules/admin/what-bonus.php:46:// @TODO Unused at the moment -./inc/modules/admin/what-config_admins.php:126: // @TODO Rewrite this to filter 'run_sqls' +./inc/modules/admin/what-config_admins.php:108: // @TODO Rewrite this to a filter +./inc/modules/admin/what-config_admins.php:136: // @TODO Rewrite this to filter 'run_sqls' ./inc/modules/admin/what-config_mods.php:55: // @TODO This can be moved into mysql-function.php, see checkModulePermissions() function ./inc/modules/admin/what-config_points.php:111: // @TODO Rewrite this to a filter ./inc/modules/admin/what-config_rallye_prices.php:195: // @TODO Rewrite these two constants diff --git a/inc/functions.php b/inc/functions.php index bf192a5fb7..79e5d8d5a7 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -819,7 +819,9 @@ function extractHostnameFromUrl (&$script) { // Extract host name $host = str_replace('http://', '', $url); - if (isInString('/', $host)) $host = substr($host, 0, strpos($host, '/')); + if (isInString('/', $host)) { + $host = substr($host, 0, strpos($host, '/')); + } // END - if // Generate relative URL //* DEBUG: */ debugOutput('SCRIPT=' . $script); @@ -832,22 +834,24 @@ function extractHostnameFromUrl (&$script) { } //* DEBUG: */ debugOutput('SCRIPT=' . $script); - if (substr($script, 0, 1) == '/') $script = substr($script, 1); + if (substr($script, 0, 1) == '/') { + $script = substr($script, 1); + } // END - if // Return host name return $host; } // Send a GET request -function sendGetRequest ($script, $data = array()) { - // Extract host name from script +function sendGetRequest ($script, $data = array(), $removeHeader = false) { + // Extract hostname and port from script $host = extractHostnameFromUrl($script); // Add data $body = http_build_query($data, '', '&'); // There should be data, else we don't need to extend $script with $body - if (empty($body)) { + if (!empty($body)) { // Do we have a question-mark in the script? if (strpos($script, '?') === false) { // No, so first char must be question mark @@ -861,7 +865,9 @@ function sendGetRequest ($script, $data = array()) { $script .= $body; // Remove trailed & to make it more conform - if (substr($script, -1, 1) == '&') $script = substr($script, 0, -1); + if (substr($script, -1, 1) == '&') { + $script = substr($script, 0, -1); + } // END - if } // END - if // Generate GET request header @@ -882,19 +888,18 @@ function sendGetRequest ($script, $data = array()) { // Send the raw request $response = sendRawRequest($host, $request); + // Should we remove header lines? + if ($removeHeader === true) { + // Okay, remove them + $response = removeHttpHeaderFromResponse($response); + } // END - if + // Return the result to the caller function return $response; } // Send a POST request -function sendPostRequest ($script, $postData) { - // Is postData an array? - if (!is_array($postData)) { - // Abort here - logDebugMessage(__FUNCTION__, __LINE__, sprintf("postData is not an array. Type: %s", gettype($postData))); - return array('', '', ''); - } // END - if - +function sendPostRequest ($script, array $postData, $removeHeader = false) { // Extract host name from script $host = extractHostnameFromUrl($script); @@ -920,6 +925,12 @@ function sendPostRequest ($script, $postData) { // Send the raw request $response = sendRawRequest($host, $request); + // Should we remove header lines? + if ($removeHeader === true) { + // Okay, remove them + $response = removeHttpHeaderFromResponse($response); + } // END - if + // Return the result to the caller function return $response; } @@ -927,7 +938,11 @@ function sendPostRequest ($script, $postData) { // Sends a raw request to another host function sendRawRequest ($host, $request) { // Init errno and errdesc with 'all fine' values - $errno = '0'; $errdesc = ''; + $errno = '0'; + $errdesc = ''; + + // Default port is 80 + $port = 80; // Initialize array $response = array('', '', ''); @@ -944,6 +959,17 @@ function sendRawRequest ($host, $request) { // Load include loadIncludeOnce('inc/classes/resolver.class.php'); + // Extract port part from host + $portArray = explode(':', $host); + if (count($portArray) == 2) { + // Extract host and port + $host = $portArray[0]; + $port = $portArray[1]; + } elseif (count($portArray) > 2) { + // This should not happen! + debug_report_bug(__FUNCTION__, __LINE__, 'Invalid ' . $host . '. Please report this to the Mailer-Project team.'); + } + // Get resolver instance $resolver = new HostnameResolver(); @@ -960,7 +986,7 @@ function sendRawRequest ($host, $request) { $ip = $resolver->resolveHostname($host); // Connect to host directly - $fp = fsockopen($ip, 80, $errno, $errdesc, 30); + $fp = fsockopen($ip, $port, $errno, $errdesc, 30); } // Is there a link? @@ -977,7 +1003,7 @@ function sendRawRequest ($host, $request) { // Do we use proxy? if ($useProxy === true) { // Setup proxy tunnel - $response = setupProxyTunnel($host, $fp); + $response = setupProxyTunnel($host, $port, $fp); // If the response is invalid, abort if ((count($response) == 3) && (empty($response[0])) && (empty($response[1])) && (empty($response[2]))) { @@ -1020,7 +1046,7 @@ function sendRawRequest ($host, $request) { } // END - if // Add it to response - $response[] = trim($line); + $response[] = $line; } // END - while // Close socket @@ -1067,6 +1093,9 @@ function sendRawRequest ($host, $request) { // Not found / access forbidden logDebugMessage(__FUNCTION__, __LINE__, 'Unexpected status code ' . $response[0] . ' detected. "200 OK" was expected.'); $response = array('', '', ''); + } else { + // Check array for chuncked encoding + $response = unchunkHttpResponse($response); } // END - if // Return response @@ -1074,12 +1103,12 @@ function sendRawRequest ($host, $request) { } // Sets up a proxy tunnel for given hostname and through resource -function setupProxyTunnel ($host, $resource) { +function setupProxyTunnel ($host, $port, $resource) { // Initialize array $response = array('', '', ''); // Generate CONNECT request header - $proxyTunnel = 'CONNECT ' . $host . ':80 HTTP/1.0' . getConfig('HTTP_EOL'); + $proxyTunnel = 'CONNECT ' . $host . ':' . $port . ' HTTP/1.0' . getConfig('HTTP_EOL'); $proxyTunnel .= 'Host: ' . $host . getConfig('HTTP_EOL'); // Use login data to proxy? (username at least!) @@ -1114,6 +1143,76 @@ function setupProxyTunnel ($host, $resource) { return $respArray; } +// Check array for chuncked encoding +function unchunkHttpResponse (array $response) { + // Default is not chunked + $isChunked = false; + + // Check if we have chunks + foreach ($response as $line) { + // Make lower-case and trim it + $line = trim(strtolower($line)); + + // Entry found? + if ((strpos($line, 'transfer-encoding') !== false) && (strpos($line, 'chunked') !== false)) { + // Found! + $isChunked = true; + break; + } // END - if + } // END - foreach + + // Is it chunked? + if ($isChunked === true) { + // Good, we still have the HTTP headers in there, so we need to get rid + // of them temporarly + //* DEBUG: */ die('
'.htmlentities(print_r(removeHttpHeaderFromResponse($response), true)).'
'); + $tempResponse = http_chunked_decode(implode('', removeHttpHeaderFromResponse($response))); + + // We got a string back from http_chunked_decode(), so we need to convert it back to an array + //* DEBUG: */ die('tempResponse['.strlen($tempResponse).']=
'.replaceReturnNewLine(htmlentities($tempResponse)).'
'); + + // Re-add the headers + $response = merge_array($GLOBALS['http_headers'], stringToArray("\n", $tempResponse)); + } // END - if + + // Return the unchunked array + return $response; +} + +// Removes HTTP header lines from a response array (e.g. output from sendRequest() ) +function removeHttpHeaderFromResponse (array $response) { + // Save headers for later usage + $GLOBALS['http_headers'] = array(); + + // The first array element has to contain HTTP + if ((isset($response[0])) && (substr(strtoupper($response[0]), 0, 5) == 'HTTP/')) { + // Okay, we have headers, now remove them with a second array + $response2 = $response; + foreach ($response as $line) { + // Remove line + array_shift($response2); + + // Add full line to temporary global array + $GLOBALS['http_headers'][] = $line; + + // Trim it for testing + $lineTest = trim($line); + + // Is this line empty? + if (empty($lineTest)) { + // Then stop here + break; + } // END - if + } // END - foreach + + // Write back the array + $response = $response2; + } // END - if + + // Return the modified response array + return $response; +} + // Taken from www.php.net isInStringIgnoreCase() user comments function isEmailValid ($email) { // Check first part of email address @@ -2507,7 +2606,7 @@ function generateAdminMailLinks ($mailType, $mailId) { if (SQL_NUMROWS($result) == 1) { // Load the entry $content = SQL_FETCHARRAY($result); - die('
'.print_r($content, true).'
'); + die(__FUNCTION__.':
content=
'.print_r($content, true).'
'); } // END - if // Free result @@ -2518,10 +2617,55 @@ function generateAdminMailLinks ($mailType, $mailId) { return $OUT; } + +/** + * determine if a string can represent a number in hexadecimal + * + * @param $hex A string to check if it is hex-encoded + * @return $foo True if the string is a hex, otherwise false + * @author Marques Johansson + * @link http://php.net/manual/en/function.http-chunked-decode.php#89786 + */ +function isHexadecimal ($hex) { + // Make it lowercase + $hex = strtolower(trim(ltrim($hex, '0'))); + + // Fix empty strings to zero + if (empty($hex)) { + $hex = 0; + } // END - if + + // Simply compare decode->encode result with original + return ($hex == dechex(hexdec($hex))); +} + +// Replace "\r" with "[r]" and "\n" with "[n]" and add a final new-line to make +// them visible to the developer. Use this function to debug e.g. buggy HTTP +// response handler functions. +function replaceReturnNewLine ($str) { + return str_replace("\r", '[r]', str_replace("\n", '[n] +', $str)); +} + +// Converts a given string by splitting it up with given delimiter similar to +// explode(), but appending the delimiter again +function stringToArray ($delimiter, $string) { + // Init array + $strArray = array(); + + // "Walk" through all entries + foreach (explode($delimiter, $string) as $split) { + // Append the delimiter and add it to the array + $strArray[] = $split . $delimiter; + } // END - foreach + + // Return array + return $strArray; +} + //----------------------------------------------------------------------------- // Automatically re-created functions, all taken from user comments on www.php.net //----------------------------------------------------------------------------- -// if (!function_exists('html_entity_decode')) { // Taken from documentation on www.php.net function html_entity_decode ($string) { @@ -2540,7 +2684,9 @@ if (!function_exists('http_build_query')) { $k = urlencode($prefix . $k); } // END - if - if ((!empty($key)) || ($key === 0)) $k = $key . '[' . urlencode($k) . ']'; + if ((!empty($key)) || ($key === 0)) { + $k = $key . '[' . urlencode($k) . ']'; + } // END - if if (is_array($v) || is_object($v)) { array_push($ret, http_build_query($v, '', $sep, $k)); @@ -2549,11 +2695,112 @@ if (!function_exists('http_build_query')) { } } // END - foreach - if (empty($sep)) $sep = ini_get('arg_separator.output'); + if (empty($sep)) { + $sep = ini_get('arg_separator.output'); + } // END - if return implode($sep, $ret); } } // END - if +if (!function_exists('http_chunked_decode')) { + /** + * dechunk an http 'transfer-encoding: chunked' message. + * + * @param $chunk The encoded message + * @return $dechunk The decoded message. If $chunk wasn't encoded properly debug_report_bug() is being called + * @author Marques Johansson + * @link http://php.net/manual/en/function.http-chunked-decode.php#89786 + */ + function http_chunked_decode ($chunk) { + // Init some variables + $offset = 0; + $len = mb_strlen($chunk); + $dechunk = ''; + + // Walk through all chunks + while ($offset < $len) { + // Where does the \r\n begin? + $lineEndAt = mb_strpos($chunk, getConfig('HTTP_EOL'), $offset); + + /* DEBUG: * + print 'lineEndAt['.__LINE__.']='.$lineEndAt.'
+offset['.__LINE__.']='.$offset.'
+len='.$len.'
+next[offset]=
'.replaceReturnNewLine(htmlentities(mb_substr($chunk, $offset, 10))).'
'; + /* DEBUG: */ + + // Get next hex-coded chunk length + $chunkLenHex = mb_substr($chunk, $offset, ($lineEndAt - $offset)); + + /* DEBUG: * + print 'chunkLenHex['.__LINE__.']='.replaceReturnNewLine(htmlentities($chunkLenHex)).'
+'; + /* DEBUG: */ + + // Validation if it is hexadecimal + if (!isHexadecimal($chunkLenHex)) { + // Please help debugging this + //* DEBUG: */ die('ABORT:chunkLenHex=
'.replaceReturnNewLine(htmlentities($chunkLenHex)).'
'); + debug_report_bug(__FUNCTION__, __LINE__, 'Value ' . $chunkLenHex . ' is not properly chunk encoded.'); + + // This won't be reached + return $chunk; + } // END - if + + // Position of next chunk is right after \r\n + $offset = $offset + strlen($chunkLenHex) + strlen(getConfig('HTTP_EOL')); + $chunkLen = hexdec(rtrim($chunkLenHex, getConfig('HTTP_EOL'))); + + /* DEBUG: * + print 'chunkLen='.$chunkLen.'
+offset['.__LINE__.']='.$offset.'
'; + /* DEBUG: */ + + // Moved out for debugging + $next = mb_substr($chunk, $offset, $chunkLen); + //* DEBUG: */ print 'next=
'.replaceReturnNewLine(htmlentities($next)).'
'; + + // Count occurrences of \r\n + $count = mb_substr_count($next, getConfig('HTTP_EOL')); + + // Correct it because we need to subtract occurrences of \r\n + $chunkLen = hexdec(rtrim($chunkLenHex, getConfig('HTTP_EOL'))) - ($count * strlen(getConfig('HTTP_EOL'))); + + $dechunk .= mb_substr($chunk, $offset, $chunkLen); + + /* DEBUG: * + print('offset['.__LINE__.']='.$offset.'
+lineEndAt['.__LINE__.']='.$lineEndAt.'
+len='.$len.'
+count='.$count.'
+chunkLen='.$chunkLen.'
+chunkLenHex='.$chunkLenHex.'
+dechunk=
'.replaceReturnNewLine(htmlentities($dechunk)).'
+chunk=
'.replaceReturnNewLine(htmlentities($chunk)).'
'); + /* DEBUG: */ + + // Is $offset + $chunkLen larger than or equal $len? + if (($offset + $chunkLen) >= $len) { + // Then stop processing here + break; + } // END - if + + // Calculate next offset of chunk + $offset = mb_strpos($chunk, getConfig('HTTP_EOL'), $offset + $chunkLen) + 2; + + /* DEBUG: * + print('offset['.__LINE__.']='.$offset.'
+next[100]=
'.replaceReturnNewLine(htmlentities(mb_substr($chunk, $offset, 100))).'
+---:---:---:---:---:---:---:---:---
+'); + /* DEBUG: */ + } // END - while + + // Return de-chunked string + return $dechunk; + } +} // END - if + // [EOF] ?> diff --git a/inc/libs/yoomedia_functions.php b/inc/libs/yoomedia_functions.php index 814db3dbb4..4307dc6bda 100644 --- a/inc/libs/yoomedia_functions.php +++ b/inc/libs/yoomedia_functions.php @@ -160,17 +160,7 @@ function YOOMEDIA_PARSE_RESPONSE ($response, $type) { $result = array(); // Cut off the header - $dummy = $response; - foreach ($response as $line) { - // Remove line - array_shift($dummy); - - // Is this line empty? - if (empty($line)) { - // Then stop here - break; - } // END - if - } // END - foreach + $dummy = removeHttpHeaderFromResponse($response); // If we have no result, abort here if (count($dummy) == 0) { @@ -183,7 +173,9 @@ function YOOMEDIA_PARSE_RESPONSE ($response, $type) { $responseLine = trim(implode("\n", $dummy)); // Last line should never be a pipe! - if (substr($responseLine, -1, 1) == '|') $responseLine = substr($responseLine, 0, -1); + if (substr($responseLine, -1, 1) == '|') { + $responseLine = substr($responseLine, 0, -1); + } // END - if // Now, explode all in one array $dataArray = explode('|', $responseLine); diff --git a/inc/modules/admin/what-updates.php b/inc/modules/admin/what-updates.php index 46049de543..b0b07ae95f 100644 --- a/inc/modules/admin/what-updates.php +++ b/inc/modules/admin/what-updates.php @@ -56,21 +56,7 @@ if (empty($response[0]) && empty($response[1]) && empty($response[2]) && empty($ // Analyse header for response code if (isInStringIgnoreCase('200 OK', $response[0])) { // Found, kill header - $pos = '0'; - foreach ($response as $k => $v) { - $v = trim($v); - if (empty($v)) { - // Header ends here (+1) - $pos = $k + 1; break; - } // END - if - } // END - foreach - - $response2 = array(); - for($i = $pos; $i < count($response); $i++) { - $response2[] = trim($response[$i]); - } - $response = $response2; unset($response2); - unset($pos); + $response = removeHttpHeaderFromResponse($response); // Which is the latest version on server? $ONLINE = array( -- 2.30.2