X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=include%2FScrape.php;h=95a3c0221f51ac32ca89017e4eeb52fceaa24101;hb=b0548018d82cf6ecc717b3382ca5e699631458e9;hp=93d68be22b892af7f95ae34bab506617b443204d;hpb=6e981ed798f8938ce7f0de91d523b1ba6359381c;p=friendica.git

diff --git a/include/Scrape.php b/include/Scrape.php
index 93d68be22b..95a3c0221f 100644
--- a/include/Scrape.php
+++ b/include/Scrape.php
@@ -2,9 +2,10 @@
 
 require_once('library/HTML5/Parser.php');
 require_once('include/crypto.php');
+require_once('include/feed.php');
 
 if(! function_exists('scrape_dfrn')) {
-function scrape_dfrn($url) {
+function scrape_dfrn($url, $dont_probe = false) {
 
 	$a = get_app();
 
@@ -17,6 +18,13 @@ function scrape_dfrn($url) {
 	if(! $s)
 		return $ret;
 
+	if (!$dont_probe) {
+		$probe = probe_url($url);
+
+		if (isset($probe["addr"]))
+			$ret["addr"] = $probe["addr"];
+	}
+
 	$headers = $a->get_curl_headers();
 	logger('scrape_dfrn: headers=' . $headers, LOGGER_DEBUG);
 
@@ -228,7 +236,9 @@ function scrape_feed($url) {
 	$a = get_app();
 
 	$ret = array();
-	$s = fetch_url($url);
+	$cookiejar = tempnam(get_temppath(), 'cookiejar-scrape-feed-');
+	$s = fetch_url($url, false, $redirects, 0, Null, $cookiejar);
+	unlink($cookiejar);
 
 	$headers = $a->get_curl_headers();
 	$code = $a->get_curl_code();
@@ -320,7 +330,7 @@ function scrape_feed($url) {
  * PROBE_DIASPORA has a bias towards returning Diaspora information
  * while PROBE_NORMAL has a bias towards dfrn/zot - in the case where
  * an address (such as a Friendica address) supports more than one type
- * of network. 
+ * of network.
  *
  */
 
@@ -357,8 +367,6 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 		$network = NETWORK_TWITTER;
 	}
 
-	// Twitter is deactivated since twitter closed its old API
-	//$twitter = ((strpos($url,'twitter.com') !== false) ? true : false);
 	$lastfm  = ((strpos($url,'last.fm/user') !== false) ? true : false);
 
 	$at_addr = ((strpos($url,'@') !== false) ? true : false);
@@ -407,7 +415,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 						$pubkey = $diaspora_key;
 					$diaspora = true;
 				}
-				if($link['@attributes']['rel'] === 'http://ostatus.org/schema/1.0/subscribe') {
+				if(($link['@attributes']['rel'] === 'http://ostatus.org/schema/1.0/subscribe') AND ($mode == PROBE_NORMAL)) {
 					$diaspora = false;
 				}
 			}
@@ -524,7 +532,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 
 
 		if(strlen($dfrn)) {
-			$ret = scrape_dfrn(($hcard) ? $hcard : $dfrn);
+			$ret = scrape_dfrn(($hcard) ? $hcard : $dfrn, true);
 			if(is_array($ret) && x($ret,'dfrn-request')) {
 				$network = NETWORK_DFRN;
 				$request = $ret['dfrn-request'];
@@ -540,9 +548,23 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 		}
 	}
 
+	// Scrape the public key from the hcard.
+	// Diaspora will remove it from the webfinger somewhere in the future.
+	if (($hcard != "") AND ($pubkey == "")) {
+		$ret = scrape_dfrn(($hcard) ? $hcard : $dfrn, true);
+		if (isset($ret["key"])) {
+			$hcard_key = $ret["key"];
+			if(strstr($hcard_key,'RSA '))
+				$pubkey = rsatopem($hcard_key);
+			else
+				$pubkey = $hcard_key;
+		}
+	}
 	if($diaspora && $diaspora_base && $diaspora_guid) {
-		if($mode == PROBE_DIASPORA || ! $notify) {
-			$notify = $diaspora_base . 'receive/users/' . $diaspora_guid;
+		$diaspora_notify = $diaspora_base.'receive/users/'.$diaspora_guid;
+
+		if($mode == PROBE_DIASPORA || ! $notify || ($notify == $diaspora_notify)) {
+			$notify = $diaspora_notify;
 			$batch  = $diaspora_base . 'receive/public' ;
 		}
 		if(strpos($url,'@'))
@@ -552,7 +574,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 	if($network !== NETWORK_ZOT && $network !== NETWORK_DFRN && $network !== NETWORK_MAIL) {
 		if($diaspora)
 			$network = NETWORK_DIASPORA;
-		elseif($has_lrdd)
+		elseif($has_lrdd AND ($notify))
 			$network  = NETWORK_OSTATUS;
 
 		if(strpos($url,'@'))
@@ -581,21 +603,6 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 			$vcard['nick'] = $addr_parts[0];
 		}
 
-		/* if($twitter) {
-			logger('twitter: setup');
-			$tid = basename($url);
-			$tapi = 'https://api.twitter.com/1/statuses/user_timeline.rss';
-			if(intval($tid))
-				$poll = $tapi . '?user_id=' . $tid;
-			else
-				$poll = $tapi . '?screen_name=' . $tid;
-			$profile = 'http://twitter.com/#!/' . $tid;
-			//$vcard['photo'] = 'https://api.twitter.com/1/users/profile_image/' . $tid;
-			$vcard['photo'] = 'https://api.twitter.com/1/users/profile_image?screen_name=' . $tid . '&size=bigger';
-			$vcard['nick'] = $tid;
-			$vcard['fn'] = $tid;
-		} */
-
 		if($lastfm) {
 			$profile = $url;
 			$poll = str_replace(array('www.','last.fm/'),array('','ws.audioscrobbler.com/1.0/'),$url) . '/recenttracks.rss';
@@ -639,106 +646,79 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 
 			if(x($feedret,'photo') && (! x($vcard,'photo')))
 				$vcard['photo'] = $feedret['photo'];
-			require_once('library/simplepie/simplepie.inc');
-			$feed = new SimplePie();
-			$xml = fetch_url($poll);
+
+			$cookiejar = tempnam(get_temppath(), 'cookiejar-scrape-feed-');
+			$xml = fetch_url($poll, false, $redirects, 0, Null, $cookiejar);
+			unlink($cookiejar);
 
 			logger('probe_url: fetch feed: ' . $poll . ' returns: ' . $xml, LOGGER_DATA);
-			$a = get_app();
-
-			logger('probe_url: scrape_feed: headers: ' . $a->get_curl_headers(), LOGGER_DATA);
-
-			// Don't try and parse an empty string
-			$feed->set_raw_data(($xml) ? $xml : '<?xml version="1.0" encoding="utf-8" ?><xml></xml>');
-
-			$feed->init();
-			if($feed->error())
-				logger('probe_url: scrape_feed: Error parsing XML: ' . $feed->error());
-
-
-			if(! x($vcard,'photo'))
-				$vcard['photo'] = $feed->get_image_url();
-			$author = $feed->get_author();
-
-			if($author) {
-				$vcard['fn'] = unxmlify(trim($author->get_name()));
-				if(! $vcard['fn'])
-					$vcard['fn'] = trim(unxmlify($author->get_email()));
-				if(strpos($vcard['fn'],'@') !== false)
-					$vcard['fn'] = substr($vcard['fn'],0,strpos($vcard['fn'],'@'));
-
-				$email = unxmlify($author->get_email());
-				if(! $profile && $author->get_link())
-					$profile = trim(unxmlify($author->get_link()));
-				if(! $vcard['photo']) {
-					$rawtags = $feed->get_feed_tags( SIMPLEPIE_NAMESPACE_ATOM_10, 'author');
-					if($rawtags) {
-						$elems = $rawtags[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10];
-						if((x($elems,'link')) && ($elems['link'][0]['attribs']['']['rel'] === 'photo'))
-							$vcard['photo'] = $elems['link'][0]['attribs']['']['href'];
-					}
-				}
-				// Fetch fullname via poco:displayName
-				$pocotags = $feed->get_feed_tags(SIMPLEPIE_NAMESPACE_ATOM_10, 'author');
-				if ($pocotags) {
-					$elems = $pocotags[0]['child']['http://portablecontacts.net/spec/1.0'];
-					if (isset($elems["displayName"]))
-						$vcard['fn'] = $elems["displayName"][0]["data"];
-					if (isset($elems["preferredUsername"]))
-						$vcard['nick'] = $elems["preferredUsername"][0]["data"];
+
+			if ($xml == "") {
+				logger("scrape_feed: XML is empty for feed ".$poll);
+				$network = NETWORK_PHANTOM;
+			} else {
+				$data = feed_import($xml,$dummy1,$dummy2, $dummy3, true);
+
+				if (!is_array($data)) {
+					logger("scrape_feed: This doesn't seem to be a feed: ".$poll);
+					$network = NETWORK_PHANTOM;
+				} else {
+					if (($vcard["photo"] == "") AND ($data["header"]["author-avatar"] != ""))
+						$vcard["photo"] = $data["header"]["author-avatar"];
+
+					if (($vcard["fn"] == "") AND ($data["header"]["author-name"] != ""))
+						$vcard["fn"] = $data["header"]["author-name"];
+
+					if (($vcard["nick"] == "") AND ($data["header"]["author-nick"] != ""))
+						$vcard["nick"] = $data["header"]["author-nick"];
+
+					if(!$profile AND ($data["header"]["author-link"] != "") AND !in_array($network, array("", NETWORK_FEED)))
+						$profile = $data["header"]["author-link"];
 				}
 			}
-			else {
-				$item = $feed->get_item(0);
-				if($item) {
-					$author = $item->get_author();
-					if($author) {
-						$vcard['fn'] = trim(unxmlify($author->get_name()));
-						if(! $vcard['fn'])
-							$vcard['fn'] = trim(unxmlify($author->get_email()));
-						if(strpos($vcard['fn'],'@') !== false)
-							$vcard['fn'] = substr($vcard['fn'],0,strpos($vcard['fn'],'@'));
-						$email = unxmlify($author->get_email());
-						if(! $profile && $author->get_link())
-							$profile = trim(unxmlify($author->get_link()));
-					}
-					if(! $vcard['photo']) {
-						$rawmedia = $item->get_item_tags('http://search.yahoo.com/mrss/','thumbnail');
-						if($rawmedia && $rawmedia[0]['attribs']['']['url'])
-							$vcard['photo'] = unxmlify($rawmedia[0]['attribs']['']['url']);
-					}
-					if(! $vcard['photo']) {
-						$rawtags = $item->get_item_tags( SIMPLEPIE_NAMESPACE_ATOM_10, 'author');
-						if($rawtags) {
-							$elems = $rawtags[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10];
-							if((x($elems,'link')) && ($elems['link'][0]['attribs']['']['rel'] === 'photo'))
-								$vcard['photo'] = $elems['link'][0]['attribs']['']['href'];
-						}
-					}
+
+			// Workaround for misconfigured Friendica servers
+			if (($network == "") AND (strstr($url, "/profile/"))) {
+				$noscrape = str_replace("/profile/", "/noscrape/", $url);
+				$noscrapejson = fetch_url($noscrape);
+				if ($noscrapejson) {
+
+					$network = NETWORK_DFRN;
+
+					$poco = str_replace("/profile/", "/poco/", $url);
+
+					$noscrapedata = json_decode($noscrapejson, true);
+
+					if (isset($noscrapedata["addr"]))
+						$addr = $noscrapedata["addr"];
+
+					if (isset($noscrapedata["fn"]))
+						$vcard["fn"] = $noscrapedata["fn"];
+
+					if (isset($noscrapedata["key"]))
+						$pubkey = $noscrapedata["key"];
+
+					if (isset($noscrapedata["photo"]))
+						$vcard["photo"] = $noscrapedata["photo"];
+
+					if (isset($noscrapedata["dfrn-request"]))
+						$request = $noscrapedata["dfrn-request"];
+
+					if (isset($noscrapedata["dfrn-confirm"]))
+						$confirm = $noscrapedata["dfrn-confirm"];
+
+					if (isset($noscrapedata["dfrn-notify"]))
+						$notify = $noscrapedata["dfrn-notify"];
+
+					if (isset($noscrapedata["dfrn-poll"]))
+						$poll = $noscrapedata["dfrn-poll"];
+
 				}
 			}
 
-			if((! $vcard['photo']) && strlen($email))
-				$vcard['photo'] = avatar_img($email);
-			if($poll === $profile)
-				$lnk = $feed->get_permalink();
-			if(isset($lnk) && strlen($lnk))
-				$profile = $lnk;
-
-			if(! $network) {
+			if(! $network)
 				$network = NETWORK_FEED;
-				// If it is a feed, don't take the author name as feed name
-				unset($vcard['fn']);
-			}
-			if(! (x($vcard,'fn')))
-				$vcard['fn'] = notags($feed->get_title());
-			if(! (x($vcard,'fn')))
-				$vcard['fn'] = notags($feed->get_description());
-
-			if(strpos($vcard['fn'],'Twitter / ') !== false) {
-				$vcard['fn'] = substr($vcard['fn'],strpos($vcard['fn'],'/')+1);
-				$vcard['fn'] = trim($vcard['fn']);
-			}
+
 			if(! x($vcard,'nick')) {
 				$vcard['nick'] = strtolower(notags(unxmlify($vcard['fn'])));
 				if(strpos($vcard['nick'],' '))
@@ -751,7 +731,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 
 	if(! x($vcard,'photo')) {
 		$a = get_app();
-		$vcard['photo'] = $a->get_baseurl() . '/images/person-175.jpg' ;
+		$vcard['photo'] = App::get_baseurl() . '/images/person-175.jpg' ;
 	}
 
 	if(! $profile)
@@ -778,6 +758,9 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 
 	$baseurl = rtrim($baseurl, "/");
 
+	if(strpos($url,'@') AND ($addr == "") AND ($network == NETWORK_DFRN))
+		$addr = str_replace('acct:', '', $url);
+
 	$vcard['fn'] = notags($vcard['fn']);
 	$vcard['nick'] = str_replace(' ','',notags($vcard['nick']));
 
@@ -820,7 +803,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 	}
 
 	// Only store into the cache if the value seems to be valid
-	if ($result['network'] != NETWORK_FEED)
+	if ($result['network'] != NETWORK_PHANTOM)
 		Cache::set("probe_url:".$mode.":".$url,serialize($result), CACHE_DAY);
 
 	return $result;