<?php
/**
- * @file src/Util/Network.php
+ * @copyright Copyright (C) 2010-2022, the Friendica project
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
*/
+
namespace Friendica\Util;
-use DOMDocument;
-use DomXPath;
use Friendica\Core\Hook;
use Friendica\Core\Logger;
-use Friendica\Core\System;
use Friendica\DI;
-use Friendica\Network\CurlResult;
+use Friendica\Model\Contact;
+use Friendica\Network\HTTPException\NotModifiedException;
+use GuzzleHttp\Psr7\Uri;
class Network
{
- /**
- * Curl wrapper
- *
- * If binary flag is true, return binary results.
- * Set the cookiejar argument to a string (e.g. "/tmp/friendica-cookies.txt")
- * to preserve cookies from one request to the next.
- *
- * @param string $url URL to fetch
- * @param bool $binary default false
- * TRUE if asked to return binary results (file download)
- * @param int $timeout Timeout in seconds, default system config value or 60 seconds
- * @param string $accept_content supply Accept: header with 'accept_content' as the value
- * @param string $cookiejar Path to cookie jar file
- * @param int $redirects The recursion counter for internal use - default 0
- *
- * @return string The fetched content
- * @throws \Friendica\Network\HTTPException\InternalServerErrorException
- */
- public static function fetchUrl(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = '', int &$redirects = 0)
- {
- $ret = self::fetchUrlFull($url, $binary, $timeout, $accept_content, $cookiejar, $redirects);
-
- return $ret->getBody();
- }
-
- /**
- * Curl wrapper with array of return values.
- *
- * Inner workings and parameters are the same as @ref fetchUrl but returns an array with
- * all the information collected during the fetch.
- *
- * @param string $url URL to fetch
- * @param bool $binary default false
- * TRUE if asked to return binary results (file download)
- * @param int $timeout Timeout in seconds, default system config value or 60 seconds
- * @param string $accept_content supply Accept: header with 'accept_content' as the value
- * @param string $cookiejar Path to cookie jar file
- * @param int $redirects The recursion counter for internal use - default 0
- *
- * @return CurlResult With all relevant information, 'body' contains the actual fetched content.
- * @throws \Friendica\Network\HTTPException\InternalServerErrorException
- */
- public static function fetchUrlFull(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = '', int &$redirects = 0)
- {
- return self::curl(
- $url,
- $binary,
- [
- 'timeout' => $timeout,
- 'accept_content' => $accept_content,
- 'cookiejar' => $cookiejar
- ],
- $redirects
- );
- }
-
- /**
- * fetches an URL.
- *
- * @param string $url URL to fetch
- * @param bool $binary default false
- * TRUE if asked to return binary results (file download)
- * @param array $opts (optional parameters) assoziative array with:
- * 'accept_content' => supply Accept: header with 'accept_content' as the value
- * 'timeout' => int Timeout in seconds, default system config value or 60 seconds
- * 'http_auth' => username:password
- * 'novalidate' => do not validate SSL certs, default is to validate using our CA list
- * 'nobody' => only return the header
- * 'cookiejar' => path to cookie jar file
- * 'header' => header array
- * @param int $redirects The recursion counter for internal use - default 0
- *
- * @return CurlResult
- * @throws \Friendica\Network\HTTPException\InternalServerErrorException
- */
- public static function curl(string $url, bool $binary = false, array $opts = [], int &$redirects = 0)
- {
- $stamp1 = microtime(true);
-
- $a = DI::app();
-
- if (strlen($url) > 1000) {
- Logger::log('URL is longer than 1000 characters. Callstack: ' . System::callstack(20), Logger::DEBUG);
- return CurlResult::createErrorCurl(substr($url, 0, 200));
- }
-
- $parts2 = [];
- $parts = parse_url($url);
- $path_parts = explode('/', $parts['path'] ?? '');
- foreach ($path_parts as $part) {
- if (strlen($part) <> mb_strlen($part)) {
- $parts2[] = rawurlencode($part);
- } else {
- $parts2[] = $part;
- }
- }
- $parts['path'] = implode('/', $parts2);
- $url = self::unparseURL($parts);
-
- if (self::isUrlBlocked($url)) {
- Logger::log('domain of ' . $url . ' is blocked', Logger::DATA);
- return CurlResult::createErrorCurl($url);
- }
-
- $ch = @curl_init($url);
-
- if (($redirects > 8) || (!$ch)) {
- return CurlResult::createErrorCurl($url);
- }
-
- @curl_setopt($ch, CURLOPT_HEADER, true);
-
- if (!empty($opts['cookiejar'])) {
- curl_setopt($ch, CURLOPT_COOKIEJAR, $opts["cookiejar"]);
- curl_setopt($ch, CURLOPT_COOKIEFILE, $opts["cookiejar"]);
- }
-
- // These settings aren't needed. We're following the location already.
- // @curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
- // @curl_setopt($ch, CURLOPT_MAXREDIRS, 5);
-
- if (!empty($opts['accept_content'])) {
- curl_setopt(
- $ch,
- CURLOPT_HTTPHEADER,
- ['Accept: ' . $opts['accept_content']]
- );
- }
-
- if (!empty($opts['header'])) {
- curl_setopt($ch, CURLOPT_HTTPHEADER, $opts['header']);
- }
-
- @curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
- @curl_setopt($ch, CURLOPT_USERAGENT, $a->getUserAgent());
-
- $range = intval(DI::config()->get('system', 'curl_range_bytes', 0));
-
- if ($range > 0) {
- @curl_setopt($ch, CURLOPT_RANGE, '0-' . $range);
- }
-
- // Without this setting it seems as if some webservers send compressed content
- // This seems to confuse curl so that it shows this uncompressed.
- /// @todo We could possibly set this value to "gzip" or something similar
- curl_setopt($ch, CURLOPT_ENCODING, '');
-
- if (!empty($opts['headers'])) {
- @curl_setopt($ch, CURLOPT_HTTPHEADER, $opts['headers']);
- }
-
- if (!empty($opts['nobody'])) {
- @curl_setopt($ch, CURLOPT_NOBODY, $opts['nobody']);
- }
-
- if (!empty($opts['timeout'])) {
- @curl_setopt($ch, CURLOPT_TIMEOUT, $opts['timeout']);
- } else {
- $curl_time = DI::config()->get('system', 'curl_timeout', 60);
- @curl_setopt($ch, CURLOPT_TIMEOUT, intval($curl_time));
- }
-
- // by default we will allow self-signed certs
- // but you can override this
-
- $check_cert = DI::config()->get('system', 'verifyssl');
- @curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false));
-
- if ($check_cert) {
- @curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2);
- }
-
- $proxy = DI::config()->get('system', 'proxy');
-
- if (strlen($proxy)) {
- @curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, 1);
- @curl_setopt($ch, CURLOPT_PROXY, $proxy);
- $proxyuser = @DI::config()->get('system', 'proxyuser');
-
- if (strlen($proxyuser)) {
- @curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyuser);
- }
- }
-
- if (DI::config()->get('system', 'ipv4_resolve', false)) {
- curl_setopt($ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
- }
-
- if ($binary) {
- @curl_setopt($ch, CURLOPT_BINARYTRANSFER, 1);
- }
-
- // don't let curl abort the entire application
- // if it throws any errors.
-
- $s = @curl_exec($ch);
- $curl_info = @curl_getinfo($ch);
-
- // Special treatment for HTTP Code 416
- // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/416
- if (($curl_info['http_code'] == 416) && ($range > 0)) {
- @curl_setopt($ch, CURLOPT_RANGE, '');
- $s = @curl_exec($ch);
- $curl_info = @curl_getinfo($ch);
- }
-
- $curlResponse = new CurlResult($url, $s, $curl_info, curl_errno($ch), curl_error($ch));
-
- if ($curlResponse->isRedirectUrl()) {
- $redirects++;
- Logger::log('curl: redirect ' . $url . ' to ' . $curlResponse->getRedirectUrl());
- @curl_close($ch);
- return self::curl($curlResponse->getRedirectUrl(), $binary, $opts, $redirects);
- }
-
- @curl_close($ch);
-
- DI::profiler()->saveTimestamp($stamp1, 'network', System::callstack());
-
- return $curlResponse;
- }
-
- /**
- * Send POST request to $url
- *
- * @param string $url URL to post
- * @param mixed $params array of POST variables
- * @param array $headers HTTP headers
- * @param int $redirects Recursion counter for internal use - default = 0
- * @param int $timeout The timeout in seconds, default system config value or 60 seconds
- *
- * @return CurlResult The content
- * @throws \Friendica\Network\HTTPException\InternalServerErrorException
- */
- public static function post(string $url, $params, array $headers = [], int $timeout = 0, int &$redirects = 0)
- {
- $stamp1 = microtime(true);
-
- if (self::isUrlBlocked($url)) {
- Logger::log('post_url: domain of ' . $url . ' is blocked', Logger::DATA);
- return CurlResult::createErrorCurl($url);
- }
-
- $a = DI::app();
- $ch = curl_init($url);
-
- if (($redirects > 8) || (!$ch)) {
- return CurlResult::createErrorCurl($url);
- }
-
- Logger::log('post_url: start ' . $url, Logger::DATA);
-
- curl_setopt($ch, CURLOPT_HEADER, true);
- curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
- curl_setopt($ch, CURLOPT_POST, 1);
- curl_setopt($ch, CURLOPT_POSTFIELDS, $params);
- curl_setopt($ch, CURLOPT_USERAGENT, $a->getUserAgent());
-
- if (DI::config()->get('system', 'ipv4_resolve', false)) {
- curl_setopt($ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
- }
-
- if (intval($timeout)) {
- curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
- } else {
- $curl_time = DI::config()->get('system', 'curl_timeout', 60);
- curl_setopt($ch, CURLOPT_TIMEOUT, intval($curl_time));
- }
-
- if (!empty($headers)) {
- curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
- }
-
- $check_cert = DI::config()->get('system', 'verifyssl');
- curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false));
-
- if ($check_cert) {
- @curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2);
- }
-
- $proxy = DI::config()->get('system', 'proxy');
-
- if (strlen($proxy)) {
- curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, 1);
- curl_setopt($ch, CURLOPT_PROXY, $proxy);
- $proxyuser = DI::config()->get('system', 'proxyuser');
- if (strlen($proxyuser)) {
- curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyuser);
- }
- }
-
- // don't let curl abort the entire application
- // if it throws any errors.
-
- $s = @curl_exec($ch);
-
- $curl_info = curl_getinfo($ch);
-
- $curlResponse = new CurlResult($url, $s, $curl_info, curl_errno($ch), curl_error($ch));
-
- if ($curlResponse->isRedirectUrl()) {
- $redirects++;
- Logger::log('post_url: redirect ' . $url . ' to ' . $curlResponse->getRedirectUrl());
- curl_close($ch);
- return self::post($curlResponse->getRedirectUrl(), $params, $headers, $redirects, $timeout);
- }
-
- curl_close($ch);
-
- DI::profiler()->saveTimestamp($stamp1, 'network', System::callstack());
-
- // Very old versions of Lighttpd don't like the "Expect" header, so we remove it when needed
- if ($curlResponse->getReturnCode() == 417) {
- $redirects++;
-
- if (empty($headers)) {
- $headers = ['Expect:'];
- } else {
- if (!in_array('Expect:', $headers)) {
- array_push($headers, 'Expect:');
- }
- }
- Logger::info('Server responds with 417, applying workaround', ['url' => $url]);
- return self::post($url, $params, $headers, $redirects, $timeout);
- }
-
- Logger::log('post_url: end ' . $url, Logger::DATA);
-
- return $curlResponse;
- }
/**
* Return raw post data from a post request
return false;
}
+ /**
+ * Checks if the provided url is on the list of domains where redirects are blocked.
+ * Returns true if it is or malformed URL, false if not.
+ *
+ * @param string $url The url to check the domain from
+ *
+ * @return boolean
+ */
+ public static function isRedirectBlocked(string $url)
+ {
+ $host = @parse_url($url, PHP_URL_HOST);
+ if (!$host) {
+ return false;
+ }
+
+ $no_redirect_list = DI::config()->get('system', 'no_redirect_list', []);
+ if (!$no_redirect_list) {
+ return false;
+ }
+
+ foreach ($no_redirect_list as $no_redirect) {
+ if (fnmatch(strtolower($no_redirect), strtolower($host))) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
/**
* Check if email address is allowed to register here.
*
Hook::callAll('avatar_lookup', $avatar);
if (! $avatar['success']) {
- $avatar['url'] = DI::baseUrl() . '/images/person-300.jpg';
+ $avatar['url'] = DI::baseUrl() . Contact::DEFAULT_AVATAR_PHOTO;
}
- Logger::log('Avatar: ' . $avatar['email'] . ' ' . $avatar['url'], Logger::DEBUG);
+ Logger::info('Avatar: ' . $avatar['email'] . ' ' . $avatar['url']);
return $avatar['url'];
}
}
/**
- * Returns the original URL of the provided URL
- *
- * This function strips tracking query params and follows redirections, either
- * through HTTP code or meta refresh tags. Stops after 10 redirections.
- *
- * @todo Remove the $fetchbody parameter that generates an extraneous HEAD request
+ * Add a missing base path (scheme and host) to a given url
*
- * @see ParseUrl::getSiteinfo
- *
- * @param string $url A user-submitted URL
- * @param int $depth The current redirection recursion level (internal)
- * @param bool $fetchbody Wether to fetch the body or not after the HEAD requests
- * @return string A canonical URL
- * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+ * @param string $url
+ * @param string $basepath
+ * @return string url
*/
- public static function finalUrl(string $url, int $depth = 1, bool $fetchbody = false)
+ public static function addBasePath(string $url, string $basepath)
{
- $a = DI::app();
-
- $url = self::stripTrackingQueryParams($url);
-
- if ($depth > 10) {
+ if (!empty(parse_url($url, PHP_URL_SCHEME)) || empty(parse_url($basepath, PHP_URL_SCHEME)) || empty($url) || empty(parse_url($url))) {
return $url;
}
- $url = trim($url, "'");
-
- $stamp1 = microtime(true);
-
- $ch = curl_init();
- curl_setopt($ch, CURLOPT_URL, $url);
- curl_setopt($ch, CURLOPT_HEADER, 1);
- curl_setopt($ch, CURLOPT_NOBODY, 1);
- curl_setopt($ch, CURLOPT_TIMEOUT, 10);
- curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
- curl_setopt($ch, CURLOPT_USERAGENT, $a->getUserAgent());
-
- curl_exec($ch);
- $curl_info = @curl_getinfo($ch);
- $http_code = $curl_info['http_code'];
- curl_close($ch);
+ $base = ['scheme' => parse_url($basepath, PHP_URL_SCHEME),
+ 'host' => parse_url($basepath, PHP_URL_HOST)];
- DI::profiler()->saveTimestamp($stamp1, "network", System::callstack());
-
- if ($http_code == 0) {
- return $url;
- }
-
- if (in_array($http_code, ['301', '302'])) {
- if (!empty($curl_info['redirect_url'])) {
- return self::finalUrl($curl_info['redirect_url'], ++$depth, $fetchbody);
- } elseif (!empty($curl_info['location'])) {
- return self::finalUrl($curl_info['location'], ++$depth, $fetchbody);
- }
- }
-
- // Check for redirects in the meta elements of the body if there are no redirects in the header.
- if (!$fetchbody) {
- return(self::finalUrl($url, ++$depth, true));
- }
-
- // if the file is too large then exit
- if ($curl_info["download_content_length"] > 1000000) {
- return $url;
- }
-
- // if it isn't a HTML file then exit
- if (!empty($curl_info["content_type"]) && !strstr(strtolower($curl_info["content_type"]), "html")) {
- return $url;
- }
-
- $stamp1 = microtime(true);
-
- $ch = curl_init();
- curl_setopt($ch, CURLOPT_URL, $url);
- curl_setopt($ch, CURLOPT_HEADER, 0);
- curl_setopt($ch, CURLOPT_NOBODY, 0);
- curl_setopt($ch, CURLOPT_TIMEOUT, 10);
- curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
- curl_setopt($ch, CURLOPT_USERAGENT, $a->getUserAgent());
-
- $body = curl_exec($ch);
- curl_close($ch);
-
- DI::profiler()->saveTimestamp($stamp1, "network", System::callstack());
-
- if (trim($body) == "") {
- return $url;
- }
-
- // Check for redirect in meta elements
- $doc = new DOMDocument();
- @$doc->loadHTML($body);
-
- $xpath = new DomXPath($doc);
-
- $list = $xpath->query("//meta[@content]");
- foreach ($list as $node) {
- $attr = [];
- if ($node->attributes->length) {
- foreach ($node->attributes as $attribute) {
- $attr[$attribute->name] = $attribute->value;
- }
- }
-
- if (@$attr["http-equiv"] == 'refresh') {
- $path = $attr["content"];
- $pathinfo = explode(";", $path);
- foreach ($pathinfo as $value) {
- if (substr(strtolower($value), 0, 4) == "url=") {
- return self::finalUrl(substr($value, 4), ++$depth);
- }
- }
- }
- }
-
- return $url;
+ $parts = array_merge($base, parse_url('/' . ltrim($url, '/')));
+ return self::unparseURL($parts);
}
/**
*
* @param array $parsed URL parts
*
- * @return string The glued URL
+ * @return string The glued URL.
+ * @deprecated since version 2021.12, use GuzzleHttp\Psr7\Uri::fromParts($parts) instead
*/
public static function unparseURL(array $parsed)
{
(strlen($fragment) ? "#".$fragment : '');
}
+ /**
+ * Convert an URI to an IDN compatible URI
+ *
+ * @param string $uri
+ * @return string
+ */
+ public static function convertToIdn(string $uri): string
+ {
+ $parts = parse_url($uri);
+ if (!empty($parts['scheme']) && !empty($parts['host'])) {
+ $parts['host'] = idn_to_ascii($parts['host']);
+ $uri = Uri::fromParts($parts);
+ } else {
+ $parts = explode('@', $uri);
+ if (count($parts) == 2) {
+ $uri = $parts[0] . '@' . idn_to_ascii($parts[1]);
+ } else {
+ $uri = idn_to_ascii($uri);
+ }
+ }
+
+ return $uri;
+ }
/**
* Switch the scheme of an url between http and https
return self::unparseURL($parsed);
}
+
+ /**
+ * Generates ETag and Last-Modified response headers and checks them against
+ * If-None-Match and If-Modified-Since request headers if present.
+ *
+ * Blocking function, sends 304 headers and exits if check passes.
+ *
+ * @param string $etag The page etag
+ * @param string $last_modified The page last modification UTC date
+ * @throws \Exception
+ */
+ public static function checkEtagModified(string $etag, string $last_modified)
+ {
+ $last_modified = DateTimeFormat::utc($last_modified, 'D, d M Y H:i:s') . ' GMT';
+
+ /**
+ * @see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.26
+ */
+ $if_none_match = filter_input(INPUT_SERVER, 'HTTP_IF_NONE_MATCH');
+ $if_modified_since = filter_input(INPUT_SERVER, 'HTTP_IF_MODIFIED_SINCE');
+ $flag_not_modified = null;
+ if ($if_none_match) {
+ $result = [];
+ preg_match('/^(?:W\/")?([^"]+)"?$/i', $etag, $result);
+ $etagTrimmed = $result[1];
+ // Lazy exact ETag match, could check weak/strong ETags
+ $flag_not_modified = $if_none_match == '*' || strpos($if_none_match, $etagTrimmed) !== false;
+ }
+
+ if ($if_modified_since && (!$if_none_match || $flag_not_modified)) {
+ // Lazy exact Last-Modified match, could check If-Modified-Since validity
+ $flag_not_modified = $if_modified_since == $last_modified;
+ }
+
+ header('Etag: ' . $etag);
+ header('Last-Modified: ' . $last_modified);
+
+ if ($flag_not_modified) {
+ throw new NotModifiedException();
+ }
+ }
+
+ /**
+ * Check if the given URL is a local link
+ *
+ * @param string $url
+ * @return bool
+ */
+ public static function isLocalLink(string $url)
+ {
+ return (strpos(Strings::normaliseLink($url), Strings::normaliseLink(DI::baseUrl())) !== false);
+ }
+
+ /**
+ * Check if the given URL is a valid HTTP/HTTPS URL
+ *
+ * @param string $url
+ * @return bool
+ */
+ public static function isValidHttpUrl(string $url)
+ {
+ $scheme = parse_url($url, PHP_URL_SCHEME);
+ return !empty($scheme) && in_array($scheme, ['http', 'https']) && parse_url($url, PHP_URL_HOST);
+ }
}
projects / friendica.git / blobdiff