<?php
/**
- * @copyright Copyright (C) 2020, Friendica
+ * @copyright Copyright (C) 2010-2023, the Friendica project
*
* @license GNU AGPL version 3 or any later version
*
namespace Friendica\Util;
-use DOMDocument;
-use DomXPath;
use Friendica\Core\Hook;
use Friendica\Core\Logger;
-use Friendica\Core\System;
use Friendica\DI;
+use Friendica\Model\Contact;
+use Friendica\Network\HTTPClient\Client\HttpClientAccept;
+use Friendica\Network\HTTPClient\Client\HttpClientOptions;
+use Friendica\Network\HTTPException\NotModifiedException;
+use GuzzleHttp\Psr7\Uri;
+use Psr\Http\Message\UriInterface;
class Network
{
* and check DNS to see if it's real (or check if is a valid IP address)
*
* @param string $url The URL to be validated
+ *
* @return string|boolean The actual working URL, false else
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/
$url = 'http://' . $url;
}
- /// @TODO Really suppress function outcomes? Why not find them + debug them?
- $h = @parse_url($url);
+ $xrd_timeout = DI::config()->get('system', 'xrd_timeout');
+ $host = parse_url($url, PHP_URL_HOST);
- if (!empty($h['host']) && (@dns_get_record($h['host'], DNS_A + DNS_CNAME) || filter_var($h['host'], FILTER_VALIDATE_IP))) {
- return $url;
+ if (empty($host) || !(filter_var($host, FILTER_VALIDATE_IP) || @dns_get_record($host . '.', DNS_A + DNS_AAAA))) {
+ return false;
}
- return false;
+ if (in_array(parse_url($url, PHP_URL_SCHEME), ['https', 'http'])) {
+ $options = [HttpClientOptions::VERIFY => true, HttpClientOptions::TIMEOUT => $xrd_timeout];
+ try {
+ $curlResult = DI::httpClient()->head($url, $options);
+ } catch (\Exception $e) {
+ return false;
+ }
+
+ // Workaround for systems that can't handle a HEAD request. Don't retry on timeouts.
+ if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() >= 400) && !in_array($curlResult->getReturnCode(), [408, 504])) {
+ try {
+ $curlResult = DI::httpClient()->get($url, HttpClientAccept::DEFAULT, $options);
+ } catch (\Exception $e) {
+ return false;
+ }
+ }
+
+ if (!$curlResult->isSuccess()) {
+ Logger::notice('Url not reachable', ['host' => $host, 'url' => $url]);
+ return false;
+ } elseif ($curlResult->isRedirectUrl()) {
+ $url = $curlResult->getRedirectUrl();
+ }
+ }
+
+ return $url;
}
/**
* @param string $addr The email address
* @return boolean True if it's a valid email address, false if it's not
*/
- public static function isEmailDomainValid(string $addr)
+ public static function isEmailDomainValid(string $addr): bool
{
if (DI::config()->get('system', 'disable_email_validation')) {
return true;
$h = substr($addr, strpos($addr, '@') + 1);
// Concerning the @ see here: https://stackoverflow.com/questions/36280957/dns-get-record-a-temporary-server-error-occurred
- if ($h && (@dns_get_record($h, DNS_A + DNS_MX) || filter_var($h, FILTER_VALIDATE_IP))) {
+ if ($h && (@dns_get_record($h, DNS_A + DNS_AAAA + DNS_MX) || filter_var($h, FILTER_VALIDATE_IP))) {
return true;
}
if ($h && @dns_get_record($h, DNS_CNAME + DNS_MX)) {
* @param string $url URL which get tested
* @return boolean True if url is allowed otherwise return false
*/
- public static function isUrlAllowed(string $url)
+ public static function isUrlAllowed(string $url): bool
{
$h = @parse_url($url);
* @param string $url The url to check the domain from
*
* @return boolean
+ *
+ * @deprecated since 2023.03 Use isUriBlocked instead
*/
- public static function isUrlBlocked(string $url)
+ public static function isUrlBlocked(string $url): bool
{
- $host = @parse_url($url, PHP_URL_HOST);
- if (!$host) {
+ try {
+ return self::isUriBlocked(new Uri($url));
+ } catch (\Throwable $e) {
+ Logger::warning('Invalid URL', ['url' => $url]);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if the provided URI domain is on the domain blocklist.
+ *
+ * @param UriInterface $uri
+ * @return boolean
+ */
+ public static function isUriBlocked(UriInterface $uri): bool
+ {
+ if (!$uri->getHost()) {
return false;
}
}
foreach ($domain_blocklist as $domain_block) {
- if (fnmatch(strtolower($domain_block['domain']), strtolower($host))) {
+ if (fnmatch(strtolower($domain_block['domain']), strtolower($uri->getHost()))) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * Checks if the provided url is on the list of domains where redirects are blocked.
+ * Returns true if it is or malformed URL, false if not.
+ *
+ * @param string $url The url to check the domain from
+ *
+ * @return boolean
+ */
+ public static function isRedirectBlocked(string $url): bool
+ {
+ $host = @parse_url($url, PHP_URL_HOST);
+ if (!$host) {
+ return false;
+ }
+
+ $no_redirect_list = DI::config()->get('system', 'no_redirect_list', []);
+ if (!$no_redirect_list) {
+ return false;
+ }
+
+ foreach ($no_redirect_list as $no_redirect) {
+ if (fnmatch(strtolower($no_redirect), strtolower($host))) {
return true;
}
}
* or if allowed list is not configured
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/
- public static function isEmailDomainAllowed(string $email)
+ public static function isEmailDomainAllowed(string $email): bool
{
$domain = strtolower(substr($email, strpos($email, '@') + 1));
if (!$domain) {
*
* @param string $domain
* @param array $domain_list
+ *
* @return boolean
*/
- public static function isDomainAllowed(string $domain, array $domain_list)
+ public static function isDomainAllowed(string $domain, array $domain_list): bool
{
$found = false;
return $found;
}
- public static function lookupAvatarByEmail(string $email)
+ public static function lookupAvatarByEmail(string $email): string
{
$avatar['size'] = 300;
$avatar['email'] = $email;
Hook::callAll('avatar_lookup', $avatar);
if (! $avatar['success']) {
- $avatar['url'] = DI::baseUrl() . '/images/person-300.jpg';
+ $avatar['url'] = DI::baseUrl() . Contact::DEFAULT_AVATAR_PHOTO;
}
- Logger::log('Avatar: ' . $avatar['email'] . ' ' . $avatar['url'], Logger::DEBUG);
+ Logger::info('Avatar: ' . $avatar['email'] . ' ' . $avatar['url']);
return $avatar['url'];
}
* Remove Google Analytics and other tracking platforms params from URL
*
* @param string $url Any user-submitted URL that may contain tracking params
+ *
* @return string The same URL stripped of tracking parameters
*/
- public static function stripTrackingQueryParams(string $url)
+ public static function stripTrackingQueryParams(string $url): string
{
$urldata = parse_url($url);
- if (!empty($urldata["query"])) {
- $query = $urldata["query"];
+
+ if (!empty($urldata['query'])) {
+ $query = $urldata['query'];
parse_str($query, $querydata);
if (is_array($querydata)) {
if (in_array(
$param,
[
- "utm_source", "utm_medium", "utm_term", "utm_content", "utm_campaign",
- "wt_mc", "pk_campaign", "pk_kwd", "mc_cid", "mc_eid",
- "fb_action_ids", "fb_action_types", "fb_ref",
- "awesm", "wtrid",
- "woo_campaign", "woo_source", "woo_medium", "woo_content", "woo_term"]
+ 'utm_source', 'utm_medium', 'utm_term', 'utm_content', 'utm_campaign',
+ // As seen from Purism
+ 'mtm_source', 'mtm_medium', 'mtm_term', 'mtm_content', 'mtm_campaign',
+ 'wt_mc', 'pk_campaign', 'pk_kwd', 'mc_cid', 'mc_eid',
+ 'fb_action_ids', 'fb_action_types', 'fb_ref',
+ 'awesm', 'wtrid',
+ 'woo_campaign', 'woo_source', 'woo_medium', 'woo_content', 'woo_term']
)
) {
- $pair = $param . "=" . urlencode($value);
- $url = str_replace($pair, "", $url);
+ $pair = $param . '=' . urlencode($value);
+ $url = str_replace($pair, '', $url);
// Second try: if the url isn't encoded completely
- $pair = $param . "=" . str_replace(" ", "+", $value);
- $url = str_replace($pair, "", $url);
+ $pair = $param . '=' . str_replace(' ', '+', $value);
+ $url = str_replace($pair, '', $url);
- // Third try: Maybey the url isn't encoded at all
- $pair = $param . "=" . $value;
- $url = str_replace($pair, "", $url);
+ // Third try: Maybe the url isn't encoded at all
+ $pair = $param . '=' . $value;
+ $url = str_replace($pair, '', $url);
- $url = str_replace(["?&", "&&"], ["?", ""], $url);
+ $url = str_replace(['?&', '&&'], ['?', ''], $url);
}
}
}
- if (substr($url, -1, 1) == "?") {
+ if (substr($url, -1, 1) == '?') {
$url = substr($url, 0, -1);
}
}
*
* @param string $url
* @param string $basepath
+ *
* @return string url
*/
- public static function addBasePath(string $url, string $basepath)
+ public static function addBasePath(string $url, string $basepath): string
{
+ $url = trim($url);
if (!empty(parse_url($url, PHP_URL_SCHEME)) || empty(parse_url($basepath, PHP_URL_SCHEME)) || empty($url) || empty(parse_url($url))) {
return $url;
}
- $base = ['scheme' => parse_url($basepath, PHP_URL_SCHEME),
- 'host' => parse_url($basepath, PHP_URL_HOST)];
+ $base = [
+ 'scheme' => parse_url($basepath, PHP_URL_SCHEME),
+ 'host' => parse_url($basepath, PHP_URL_HOST),
+ ];
$parts = array_merge($base, parse_url('/' . ltrim($url, '/')));
return self::unparseURL($parts);
}
- /**
- * Returns the original URL of the provided URL
- *
- * This function strips tracking query params and follows redirections, either
- * through HTTP code or meta refresh tags. Stops after 10 redirections.
- *
- * @todo Remove the $fetchbody parameter that generates an extraneous HEAD request
- *
- * @see ParseUrl::getSiteinfo
- *
- * @param string $url A user-submitted URL
- * @param int $depth The current redirection recursion level (internal)
- * @param bool $fetchbody Wether to fetch the body or not after the HEAD requests
- * @return string A canonical URL
- * @throws \Friendica\Network\HTTPException\InternalServerErrorException
- */
- public static function finalUrl(string $url, int $depth = 1, bool $fetchbody = false)
- {
- $a = DI::app();
-
- $url = self::stripTrackingQueryParams($url);
-
- if ($depth > 10) {
- return $url;
- }
-
- $url = trim($url, "'");
-
- $stamp1 = microtime(true);
-
- $ch = curl_init();
- curl_setopt($ch, CURLOPT_URL, $url);
- curl_setopt($ch, CURLOPT_HEADER, 1);
- curl_setopt($ch, CURLOPT_NOBODY, 1);
- curl_setopt($ch, CURLOPT_TIMEOUT, 10);
- curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
- curl_setopt($ch, CURLOPT_USERAGENT, $a->getUserAgent());
-
- curl_exec($ch);
- $curl_info = @curl_getinfo($ch);
- $http_code = $curl_info['http_code'];
- curl_close($ch);
-
- DI::profiler()->saveTimestamp($stamp1, "network", System::callstack());
-
- if ($http_code == 0) {
- return $url;
- }
-
- if (in_array($http_code, ['301', '302'])) {
- if (!empty($curl_info['redirect_url'])) {
- return self::finalUrl($curl_info['redirect_url'], ++$depth, $fetchbody);
- } elseif (!empty($curl_info['location'])) {
- return self::finalUrl($curl_info['location'], ++$depth, $fetchbody);
- }
- }
-
- // Check for redirects in the meta elements of the body if there are no redirects in the header.
- if (!$fetchbody) {
- return(self::finalUrl($url, ++$depth, true));
- }
-
- // if the file is too large then exit
- if ($curl_info["download_content_length"] > 1000000) {
- return $url;
- }
-
- // if it isn't a HTML file then exit
- if (!empty($curl_info["content_type"]) && !strstr(strtolower($curl_info["content_type"]), "html")) {
- return $url;
- }
-
- $stamp1 = microtime(true);
-
- $ch = curl_init();
- curl_setopt($ch, CURLOPT_URL, $url);
- curl_setopt($ch, CURLOPT_HEADER, 0);
- curl_setopt($ch, CURLOPT_NOBODY, 0);
- curl_setopt($ch, CURLOPT_TIMEOUT, 10);
- curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
- curl_setopt($ch, CURLOPT_USERAGENT, $a->getUserAgent());
-
- $body = curl_exec($ch);
- curl_close($ch);
-
- DI::profiler()->saveTimestamp($stamp1, "network", System::callstack());
-
- if (trim($body) == "") {
- return $url;
- }
-
- // Check for redirect in meta elements
- $doc = new DOMDocument();
- @$doc->loadHTML($body);
-
- $xpath = new DomXPath($doc);
-
- $list = $xpath->query("//meta[@content]");
- foreach ($list as $node) {
- $attr = [];
- if ($node->attributes->length) {
- foreach ($node->attributes as $attribute) {
- $attr[$attribute->name] = $attribute->value;
- }
- }
-
- if (@$attr["http-equiv"] == 'refresh') {
- $path = $attr["content"];
- $pathinfo = explode(";", $path);
- foreach ($pathinfo as $value) {
- if (substr(strtolower($value), 0, 4) == "url=") {
- return self::finalUrl(substr($value, 4), ++$depth);
- }
- }
- }
- }
-
- return $url;
- }
-
/**
* Find the matching part between two url
*
* @param string $url1
* @param string $url2
- * @return string The matching part
+ *
+ * @return string The matching part or empty string on error
*/
- public static function getUrlMatch(string $url1, string $url2)
+ public static function getUrlMatch(string $url1, string $url2): string
{
- if (($url1 == "") || ($url2 == "")) {
- return "";
+ if (($url1 == '') || ($url2 == '')) {
+ return '';
}
$url1 = Strings::normaliseLink($url1);
$parts1 = parse_url($url1);
$parts2 = parse_url($url2);
- if (!isset($parts1["host"]) || !isset($parts2["host"])) {
- return "";
+ if (!isset($parts1['host']) || !isset($parts2['host'])) {
+ return '';
}
- if (empty($parts1["scheme"])) {
- $parts1["scheme"] = '';
+ if (empty($parts1['scheme'])) {
+ $parts1['scheme'] = '';
}
- if (empty($parts2["scheme"])) {
- $parts2["scheme"] = '';
+ if (empty($parts2['scheme'])) {
+ $parts2['scheme'] = '';
}
- if ($parts1["scheme"] != $parts2["scheme"]) {
- return "";
+ if ($parts1['scheme'] != $parts2['scheme']) {
+ return '';
}
- if (empty($parts1["host"])) {
- $parts1["host"] = '';
+ if (empty($parts1['host'])) {
+ $parts1['host'] = '';
}
- if (empty($parts2["host"])) {
- $parts2["host"] = '';
+ if (empty($parts2['host'])) {
+ $parts2['host'] = '';
}
- if ($parts1["host"] != $parts2["host"]) {
- return "";
+ if ($parts1['host'] != $parts2['host']) {
+ return '';
}
- if (empty($parts1["port"])) {
- $parts1["port"] = '';
+ if (empty($parts1['port'])) {
+ $parts1['port'] = '';
}
- if (empty($parts2["port"])) {
- $parts2["port"] = '';
+ if (empty($parts2['port'])) {
+ $parts2['port'] = '';
}
- if ($parts1["port"] != $parts2["port"]) {
- return "";
+ if ($parts1['port'] != $parts2['port']) {
+ return '';
}
- $match = $parts1["scheme"]."://".$parts1["host"];
+ $match = $parts1['scheme'] . '://' . $parts1['host'];
- if ($parts1["port"]) {
- $match .= ":".$parts1["port"];
+ if ($parts1['port']) {
+ $match .= ':' . $parts1['port'];
}
- if (empty($parts1["path"])) {
- $parts1["path"] = '';
+ if (empty($parts1['path'])) {
+ $parts1['path'] = '';
}
- if (empty($parts2["path"])) {
- $parts2["path"] = '';
+ if (empty($parts2['path'])) {
+ $parts2['path'] = '';
}
- $pathparts1 = explode("/", $parts1["path"]);
- $pathparts2 = explode("/", $parts2["path"]);
+ $pathparts1 = explode('/', $parts1['path']);
+ $pathparts2 = explode('/', $parts2['path']);
$i = 0;
- $path = "";
+ $path = '';
do {
$path1 = $pathparts1[$i] ?? '';
$path2 = $pathparts2[$i] ?? '';
if ($path1 == $path2) {
- $path .= $path1."/";
+ $path .= $path1 . '/';
}
} while (($path1 == $path2) && ($i++ <= count($pathparts1)));
*
* @param array $parsed URL parts
*
- * @return string The glued URL
+ * @return string|null The glued URL or null on error
+ * @deprecated since version 2021.12, use GuzzleHttp\Psr7\Uri::fromParts($parts) instead
*/
- public static function unparseURL(array $parsed)
+ public static function unparseURL(array $parsed): string
{
$get = function ($key) use ($parsed) {
return isset($parsed[$key]) ? $parsed[$key] : null;
$scheme = $get('scheme');
$query = $get('query');
$fragment = $get('fragment');
- $authority = ($userinfo !== null ? $userinfo."@" : '') .
+ $authority = ($userinfo !== null ? $userinfo . '@' : '') .
$get('host') .
($port ? ":$port" : '');
- return (strlen($scheme) ? $scheme.":" : '') .
- (strlen($authority) ? "//".$authority : '') .
+ return (!empty($scheme) ? $scheme . ':' : '') .
+ (!empty($authority) ? '//' . $authority : '') .
$get('path') .
- (strlen($query) ? "?".$query : '') .
- (strlen($fragment) ? "#".$fragment : '');
+ (!empty($query) ? '?' . $query : '') .
+ (!empty($fragment) ? '#' . $fragment : '');
}
+ /**
+ * Convert an URI to an IDN compatible URI
+ *
+ * @param string $uri
+ *
+ * @return string
+ */
+ public static function convertToIdn(string $uri): string
+ {
+ $parts = parse_url($uri);
+ if (!empty($parts['scheme']) && !empty($parts['host'])) {
+ $parts['host'] = idn_to_ascii($parts['host']);
+ $uri = (string)Uri::fromParts($parts);
+ } else {
+ $parts = explode('@', $uri);
+ if (count($parts) == 2) {
+ $uri = $parts[0] . '@' . idn_to_ascii($parts[1]);
+ } else {
+ $uri = idn_to_ascii($uri);
+ }
+ }
+
+ return $uri;
+ }
/**
* Switch the scheme of an url between http and https
*
- * @param string $url URL
+ * @param string $url
*
- * @return string switched URL
+ * @return string Switched URL
*/
- public static function switchScheme(string $url)
+ public static function switchScheme(string $url): string
{
$scheme = parse_url($url, PHP_URL_SCHEME);
if (empty($scheme)) {
*
* @param string $path
* @param array $additionalParams Associative array of parameters
+ *
* @return string
*/
- public static function appendQueryParam(string $path, array $additionalParams)
+ public static function appendQueryParam(string $path, array $additionalParams): string
{
$parsed = parse_url($path);
*
* @param string $etag The page etag
* @param string $last_modified The page last modification UTC date
+ *
+ * @return void
* @throws \Exception
*/
public static function checkEtagModified(string $etag, string $last_modified)
header('Last-Modified: ' . $last_modified);
if ($flag_not_modified) {
- header("HTTP/1.1 304 Not Modified");
- exit;
+ throw new NotModifiedException();
}
}
+
+ /**
+ * Check if the given URL is a local link
+ *
+ * @param string $url
+ *
+ * @return bool
+ * @deprecated since 2023.09, please use BaseUrl->isLocalUrl or BaseUrl->isLocalUri instead.
+ */
+ public static function isLocalLink(string $url): bool
+ {
+ return DI::baseUrl()->isLocalUrl($url);
+ }
+
+ /**
+ * Check if the given URL is a valid HTTP/HTTPS URL
+ *
+ * @param string $url
+ * @return bool
+ */
+ public static function isValidHttpUrl(string $url): bool
+ {
+ $scheme = parse_url($url, PHP_URL_SCHEME);
+ return !empty($scheme) && in_array($scheme, ['http', 'https']) && parse_url($url, PHP_URL_HOST);
+ }
}