Merge https://github.com/friendica/friendica into pull

[friendica.git] / include / network.php

diff --git a/include/network.php b/include/network.php
index 310b988cb3460fcee23b4af08be4e83ae6688d8c..4c6af8e71b45e0e599b4dc7c5c67394f33f67f79 100644 (file)
--- a/include/network.php
+++ b/include/network.php
@@ -35,8 +35,7 @@ function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_
        }
 
        @curl_setopt($ch, CURLOPT_RETURNTRANSFER,true);
-       //@curl_setopt($ch, CURLOPT_USERAGENT, "Friendica");
-       @curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; Friendica)");
+       @curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; ".FRIENDICA_PLATFORM." ".FRIENDICA_VERSION."-".DB_UPDATE_VERSION.")");
 
 
        if(intval($timeout)) {
@@ -73,7 +72,7 @@ function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_
        $base = $s;
        $curl_info = @curl_getinfo($ch);
        $http_code = $curl_info['http_code'];
-//     logger('fetch_url:' . $http_code . ' data: ' . $s);
+       logger('fetch_url '.$url.': '.$http_code." ".$s, LOGGER_DATA);
        $header = '';
 
        // Pull out multiple headers, e.g. proxy and continuation headers
@@ -122,7 +121,6 @@ function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_
 
 if(! function_exists('post_url')) {
 function post_url($url,$params, $headers = null, &$redirects = 0, $timeout = 0) {
-
        $stamp1 = microtime(true);
 
        $a = get_app();
@@ -130,11 +128,13 @@ function post_url($url,$params, $headers = null, &$redirects = 0, $timeout = 0)
        if(($redirects > 8) || (! $ch))
                return false;
 
+       logger("post_url: start ".$url, LOGGER_DATA);
+
        curl_setopt($ch, CURLOPT_HEADER, true);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER,true);
        curl_setopt($ch, CURLOPT_POST,1);
        curl_setopt($ch, CURLOPT_POSTFIELDS,$params);
-       curl_setopt($ch, CURLOPT_USERAGENT, "Friendica");
+       curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; ".FRIENDICA_PLATFORM." ".FRIENDICA_VERSION."-".DB_UPDATE_VERSION.")");
 
        if(intval($timeout)) {
                curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
@@ -178,6 +178,8 @@ function post_url($url,$params, $headers = null, &$redirects = 0, $timeout = 0)
        $curl_info = curl_getinfo($ch);
        $http_code = $curl_info['http_code'];
 
+       logger("post_url: result ".$http_code." - ".$url, LOGGER_DATA);
+
        $header = '';
 
        // Pull out multiple headers, e.g. proxy and continuation headers
@@ -190,16 +192,18 @@ function post_url($url,$params, $headers = null, &$redirects = 0, $timeout = 0)
        }
 
        if($http_code == 301 || $http_code == 302 || $http_code == 303 || $http_code == 307) {
-        $matches = array();
-        preg_match('/(Location:|URI:)(.*?)\n/', $header, $matches);
-        $newurl = trim(array_pop($matches));
+               $matches = array();
+               preg_match('/(Location:|URI:)(.*?)\n/', $header, $matches);
+               $newurl = trim(array_pop($matches));
                if(strpos($newurl,'/') === 0)
                        $newurl = $old_location_info["scheme"] . "://" . $old_location_info["host"] . $newurl;
-        if (filter_var($newurl, FILTER_VALIDATE_URL)) {
-            $redirects++;
-            return fetch_url($newurl,false,$redirects,$timeout);
-        }
-    }
+               if (filter_var($newurl, FILTER_VALIDATE_URL)) {
+                       $redirects++;
+                       logger("post_url: redirect ".$url." to ".$newurl);
+                       return post_url($newurl,$params, $headers, $redirects, $timeout);
+                       //return fetch_url($newurl,false,$redirects,$timeout);
+               }
+       }
        $a->set_curl_code($http_code);
        $body = substr($s,strlen($header));
 
@@ -209,6 +213,8 @@ function post_url($url,$params, $headers = null, &$redirects = 0, $timeout = 0)
 
        $a->save_timestamp($stamp1, "network");
 
+       logger("post_url: end ".$url, LOGGER_DATA);
+
        return($body);
 }}
 
@@ -1107,8 +1113,41 @@ function xml2array($contents, $namespaces = true, $get_attributes=1, $priority =
 }
 
 function original_url($url, $depth=1, $fetchbody = false) {
+
+       // Remove Analytics Data from Google and other tracking platforms
+       $urldata = parse_url($url);
+       if (is_string($urldata["query"])) {
+               $query = $urldata["query"];
+               parse_str($query, $querydata);
+
+               if (is_array($querydata))
+                       foreach ($querydata AS $param=>$value)
+                               if (in_array($param, array("utm_source", "utm_medium", "utm_term", "utm_content", "utm_campaign",
+                                                       "wt_mc", "pk_campaign", "pk_kwd", "mc_cid", "mc_eid",
+                                                       "fb_action_ids", "fb_action_types", "fb_ref",
+                                                       "awesm",
+                                                       "woo_campaign", "woo_source", "woo_medium", "woo_content", "woo_term"))) {
+
+                                       $pair = $param."=".urlencode($value);
+                                       $url = str_replace($pair, "", $url);
+
+                                       // Second try: if the url isn't encoded completely
+                                       $pair = $param."=".str_replace(" ", "+", $value);
+                                       $url = str_replace($pair, "", $url);
+
+                                       // Third try: Maybey the url isn't encoded at all
+                                       $pair = $param."=".$value;
+                                       $url = str_replace($pair, "", $url);
+
+                                       $url = str_replace(array("?&", "&&"), array("?", ""), $url);
+                               }
+
+               if (substr($url, -1, 1) == "?")
+                       $url = substr($url, 0, -1);
+       }
+
         if ($depth > 10)
-                return($url);
+               return($url);
 
         $url = trim($url, "'");
 
@@ -1124,7 +1163,7 @@ function original_url($url, $depth=1, $fetchbody = false) {
 
         curl_setopt($ch, CURLOPT_TIMEOUT, 10);
         curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
-        curl_setopt($ch,CURLOPT_USERAGENT,'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20100101 Firefox/24.0');
+       curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; ".FRIENDICA_PLATFORM." ".FRIENDICA_VERSION."-".DB_UPDATE_VERSION.")");
 
         $header = curl_exec($ch);
         $curl_info = @curl_getinfo($ch);