-- ------------------------------------------
-- Friendica 2019.12-rc (Dalmatian Bellflower)
--- DB_UPDATE_VERSION 1326
+-- DB_UPDATE_VERSION 1328
-- ------------------------------------------
`info` text COMMENT '',
`register_policy` tinyint NOT NULL DEFAULT 0 COMMENT '',
`registered-users` int unsigned NOT NULL DEFAULT 0 COMMENT 'Number of registered users',
+ `directory-type` tinyint DEFAULT 0 COMMENT 'Type of directory service (Poco, Mastodon)',
`poco` varchar(255) NOT NULL DEFAULT '' COMMENT '',
`noscrape` varchar(255) NOT NULL DEFAULT '' COMMENT '',
`network` char(4) NOT NULL DEFAULT '' COMMENT '',
DBA::close($data);
// Add found profiles from the global directory to the local directory
- Worker::add(PRIORITY_LOW, 'DiscoverPoCo', "dirsearch", urlencode($search));
+ Worker::add(PRIORITY_LOW, 'SearchDirectory', $search);
return $resultList;
}
public static function getId($contact)
{
$gcontact_id = 0;
- $doprobing = false;
- $last_failure_str = '';
- $last_contact_str = '';
if (empty($contact['network'])) {
Logger::notice('Empty network', ['url' => $contact['url'], 'callstack' => System::callstack()]);
$gcnt = DBA::selectFirst('gcontact', $fields, ['nurl' => Strings::normaliseLink($contact['url'])]);
if (DBA::isResult($gcnt)) {
$gcontact_id = $gcnt['id'];
-
- // Update every 90 days
- if (empty($gcnt['network']) || in_array($gcnt['network'], Protocol::FEDERATED)) {
- $last_failure_str = $gcnt['last_failure'];
- $last_failure = strtotime($gcnt['last_failure']);
- $last_contact_str = $gcnt['last_contact'];
- $last_contact = strtotime($gcnt['last_contact']);
- $doprobing = (((time() - $last_contact) > (90 * 86400)) && ((time() - $last_failure) > (90 * 86400)));
- }
} else {
$contact['location'] = $contact['location'] ?? '';
$contact['about'] = $contact['about'] ?? '';
$cnt = DBA::selectFirst('gcontact', ['id', 'network'], $condition, ['order' => ['id']]);
if (DBA::isResult($cnt)) {
$gcontact_id = $cnt['id'];
- $doprobing = (empty($cnt['network']) || in_array($cnt['network'], Protocol::FEDERATED));
}
}
DBA::unlock();
- if ($doprobing) {
- Logger::notice('Probing', ['contact' => $last_contact_str, "failure" => $last_failure_str, "checking" => $contact['url']]);
- Worker::add(PRIORITY_LOW, 'GProbe', $contact['url']);
- }
-
return $gcontact_id;
}
return;
}
- if (!$force && !PortableContact::updateNeeded($gcontact['created'], $gcontact['updated'], $gcontact['last_failure'], $gcontact['last_contact'])) {
+ if (!$force && !GServer::updateNeeded($gcontact['created'], $gcontact['updated'], $gcontact['last_failure'], $gcontact['last_contact'])) {
Logger::info("Don't update profile", ['url' => $data['url'], 'updated' => $gcontact['updated']]);
return;
}
*/
class GServer
{
+ // Directory types
+ const DT_NONE = 0;
+ const DT_POCO = 1;
+ const DT_MASTODON = 2;
/**
* Checks if the given server is reachable
*
return self::check($server, $network, $force);
}
+ /**
+ * Decides if a server needs to be updated, based upon several date fields
+ *
+ * @param date $created Creation date of that server entry
+ * @param date $updated When had the server entry be updated
+ * @param date $last_failure Last failure when contacting that server
+ * @param date $last_contact Last time the server had been contacted
+ *
+ * @return boolean Does the server record needs an update?
+ */
+ public static function updateNeeded($created, $updated, $last_failure, $last_contact)
+ {
+ $now = strtotime(DateTimeFormat::utcNow());
+
+ if ($updated > $last_contact) {
+ $contact_time = strtotime($updated);
+ } else {
+ $contact_time = strtotime($last_contact);
+ }
+
+ $failure_time = strtotime($last_failure);
+ $created_time = strtotime($created);
+
+ // If there is no "created" time then use the current time
+ if ($created_time <= 0) {
+ $created_time = $now;
+ }
+
+ // If the last contact was less than 24 hours then don't update
+ if (($now - $contact_time) < (60 * 60 * 24)) {
+ return false;
+ }
+
+ // If the last failure was less than 24 hours then don't update
+ if (($now - $failure_time) < (60 * 60 * 24)) {
+ return false;
+ }
+
+ // If the last contact was less than a week ago and the last failure is older than a week then don't update
+ //if ((($now - $contact_time) < (60 * 60 * 24 * 7)) && ($contact_time > $failure_time))
+ // return false;
+
+ // If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week
+ if ((($now - $contact_time) > (60 * 60 * 24 * 7)) && (($now - $created_time) > (60 * 60 * 24 * 7)) && (($now - $failure_time) < (60 * 60 * 24 * 7))) {
+ return false;
+ }
+
+ // If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month
+ if ((($now - $contact_time) > (60 * 60 * 24 * 30)) && (($now - $created_time) > (60 * 60 * 24 * 30)) && (($now - $failure_time) < (60 * 60 * 24 * 30))) {
+ return false;
+ }
+
+ return true;
+ }
+
/**
* Checks the state of the given server.
*
$last_failure = DBA::NULL_DATETIME;
}
- if (!$force && !PortableContact::updateNeeded($gserver['created'], '', $last_failure, $last_contact)) {
+ if (!$force && !self::updateNeeded($gserver['created'], '', $last_failure, $last_contact)) {
Logger::info('No update needed', ['server' => $server_url]);
return ($last_contact >= $last_failure);
}
*/
public static function detect(string $url, string $network = '')
{
+ Logger::info('Detect server type', ['server' => $url]);
$serverdata = [];
+ $original_url = $url;
+
+ // Remove URL content that is not supposed to exist for a server url
+ $urlparts = parse_url($url);
+ unset($urlparts['user']);
+ unset($urlparts['pass']);
+ unset($urlparts['query']);
+ unset($urlparts['fragment']);
+ $url = Network::unparseURL($urlparts);
+
+ // If the URL missmatches, then we mark the old entry as failure
+ if ($url != $original_url) {
+ DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($original_url)]);
+ }
+
// When a nodeinfo is present, we don't need to dig further
$xrd_timeout = Config::get('system', 'xrd_timeout');
$curlResult = Network::curl($url . '/.well-known/nodeinfo', false, ['timeout' => $xrd_timeout]);
$serverdata = $nodeinfo;
}
+ // Detect the directory type
+ $serverdata['directory-type'] = self::DT_NONE;
$serverdata = self::checkPoCo($url, $serverdata);
+ $serverdata = self::checkMastodonDirectory($url, $serverdata);
// We can't detect the network type. Possibly it is some system that we don't know yet
if (empty($serverdata['network'])) {
*/
private static function checkPoCo(string $url, array $serverdata)
{
+ $serverdata['poco'] = '';
+
$curlResult = Network::curl($url. '/poco');
if (!$curlResult->isSuccess()) {
return $serverdata;
if (!empty($data['totalResults'])) {
$registeredUsers = $serverdata['registered-users'] ?? 0;
$serverdata['registered-users'] = max($data['totalResults'], $registeredUsers);
+ $serverdata['directory-type'] = self::DT_POCO;
$serverdata['poco'] = $url . '/poco';
- } else {
- $serverdata['poco'] = '';
+ }
+
+ return $serverdata;
+ }
+
+ /**
+ * Checks if the given server does have a Mastodon style directory endpoint.
+ *
+ * @param string $url URL of the given server
+ * @param array $serverdata array with server data
+ *
+ * @return array server data
+ */
+ public static function checkMastodonDirectory(string $url, array $serverdata)
+ {
+ $curlResult = Network::curl($url . '/api/v1/directory?limit=1');
+ if (!$curlResult->isSuccess()) {
+ return $serverdata;
+ }
+
+ $data = json_decode($curlResult->getBody(), true);
+ if (empty($data)) {
+ return $serverdata;
+ }
+
+ if (count($data) == 1) {
+ $serverdata['directory-type'] = self::DT_MASTODON;
}
return $serverdata;
}
return $serverdata;
}
+
+ /**
+ * Update the user directory of a given gserver record
+ *
+ * @param array $gserver gserver record
+ */
+ public static function updateDirectory(array $gserver)
+ {
+ /// @todo Add Mastodon API directory
+
+ if (!empty($gserver['poco'])) {
+ PortableContact::discoverSingleServer($gserver['id']);
+ }
+ }
}
Logger::log('Not authenticated. Invoking reverse magic-auth for ' . $my_url, Logger::DEBUG);
- Worker::add(PRIORITY_LOW, 'GProbe', $my_url);
-
// Remove the "addr" parameter from the destination. It is later added as separate parameter again.
$addr_request = 'addr=' . urlencode($addr);
$query = rtrim(str_replace($addr_request, '', $a->query_string), '?&');
public static function loadWorker($cid, $uid = 0, $zcid = 0, $url = null)
{
// Call the function "load" via the worker
- Worker::add(PRIORITY_LOW, "DiscoverPoCo", "load", (int)$cid, (int)$uid, (int)$zcid, $url);
+ Worker::add(PRIORITY_LOW, 'FetchPoCo', (int)$cid, (int)$uid, (int)$zcid, $url);
}
/**
return(preg_match("=https?://.+/user/\d+=ism", $url, $matches));
}
- public static function updateNeeded($created, $updated, $last_failure, $last_contact)
- {
- $now = strtotime(DateTimeFormat::utcNow());
-
- if ($updated > $last_contact) {
- $contact_time = strtotime($updated);
- } else {
- $contact_time = strtotime($last_contact);
- }
-
- $failure_time = strtotime($last_failure);
- $created_time = strtotime($created);
-
- // If there is no "created" time then use the current time
- if ($created_time <= 0) {
- $created_time = $now;
- }
-
- // If the last contact was less than 24 hours then don't update
- if (($now - $contact_time) < (60 * 60 * 24)) {
- return false;
- }
-
- // If the last failure was less than 24 hours then don't update
- if (($now - $failure_time) < (60 * 60 * 24)) {
- return false;
- }
-
- // If the last contact was less than a week ago and the last failure is older than a week then don't update
- //if ((($now - $contact_time) < (60 * 60 * 24 * 7)) && ($contact_time > $failure_time))
- // return false;
-
- // If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week
- if ((($now - $contact_time) > (60 * 60 * 24 * 7)) && (($now - $created_time) > (60 * 60 * 24 * 7)) && (($now - $failure_time) < (60 * 60 * 24 * 7))) {
- return false;
- }
-
- // If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month
- if ((($now - $contact_time) > (60 * 60 * 24 * 30)) && (($now - $created_time) > (60 * 60 * 24 * 30)) && (($now - $failure_time) < (60 * 60 * 24 * 30))) {
- return false;
- }
-
- return true;
- }
-
/**
* @brief Returns a list of all known servers
* @return array List of server urls
if (!DBA::isResult($r)) {
Logger::log("Call server check for server ".$server_url, Logger::DEBUG);
- Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $server_url);
+ Worker::add(PRIORITY_LOW, 'UpdateGServer', $server_url);
}
}
}
if (!empty($servers['pods'])) {
foreach ($servers['pods'] as $server) {
- Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", "https://" . $server['host']);
+ Worker::add(PRIORITY_LOW, 'UpdateGServer', 'https://' . $server['host']);
}
}
}
foreach ($servers['instances'] as $server) {
$url = (is_null($server['https_score']) ? 'http' : 'https') . '://' . $server['name'];
- Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $url);
+ Worker::add(PRIORITY_LOW, 'UpdateGServer', $url);
}
}
}
$last_update = date('c', time() - (60 * 60 * 24 * $requery_days));
- $gservers = q("SELECT `id`, `url`, `nurl`, `network`
+ $gservers = q("SELECT `id`, `url`, `nurl`, `network`, `poco`
FROM `gserver`
WHERE `last_contact` >= `last_failure`
AND `poco` != ''
}
Logger::log('Update directory from server ' . $gserver['url'] . ' with ID ' . $gserver['id'], Logger::DEBUG);
- Worker::add(PRIORITY_LOW, 'DiscoverPoCo', 'update_server_directory', (int) $gserver['id']);
+ Worker::add(PRIORITY_LOW, 'UpdateServerDirectory', $gserver);
if (!$complete && ( --$no_of_queries == 0)) {
break;
// Fork the cron jobs in separate parts to avoid problems when one of them is crashing
Hook::fork($a->queue['priority'], "cron");
- // run the process to discover global contacts in the background
- Worker::add(PRIORITY_LOW, "DiscoverPoCo");
+ // run the process to update server directories in the background
+ Worker::add(PRIORITY_LOW, 'UpdateServerDirectories');
// run the process to update locally stored global contacts in the background
- Worker::add(PRIORITY_LOW, "DiscoverPoCo", "checkcontact");
+ Worker::add(PRIORITY_LOW, 'UpdateGContacts');
// Expire and remove user entries
Worker::add(PRIORITY_MEDIUM, "CronJobs", "expire_and_remove_users");
// update nodeinfo data
Worker::add(PRIORITY_LOW, "CronJobs", "nodeinfo");
- Worker::add(PRIORITY_LOW, "DiscoverPoCo", "update_server");
+ Worker::add(PRIORITY_LOW, 'UpdateGServers');
- Worker::add(PRIORITY_LOW, "DiscoverPoCo", "suggestions");
+ Worker::add(PRIORITY_LOW, 'UpdateSuggestions');
Worker::add(PRIORITY_LOW, 'Expire');
+++ /dev/null
-<?php
-/**
- * @file src/Worker/DiscoverPoCo.php
- */
-namespace Friendica\Worker;
-
-use Friendica\Core\Cache;
-use Friendica\Core\Config;
-use Friendica\Core\Logger;
-use Friendica\Core\Protocol;
-use Friendica\Core\Worker;
-use Friendica\Database\DBA;
-use Friendica\Model\GContact;
-use Friendica\Model\Contact;
-use Friendica\Model\GServer;
-use Friendica\Network\Probe;
-use Friendica\Protocol\PortableContact;
-use Friendica\Util\DateTimeFormat;
-use Friendica\Util\Network;
-use Friendica\Util\Strings;
-
-class DiscoverPoCo
-{
- /// @todo Clean up this mess of a parameter hell and split it in several classes
- public static function execute($command = '', $param1 = '', $param2 = '', $param3 = '', $param4 = '')
- {
- /*
- This function can be called in these ways:
- - dirsearch <search pattern>: Searches for "search pattern" in the directory. "search pattern" is url encoded.
- - checkcontact: Updates gcontact entries
- - suggestions: Discover other servers for their contacts.
- - server <poco url>: Searches for the poco server list. "poco url" is base64 encoded.
- - update_server: Frequently check the first 250 servers for vitality.
- - update_server_directory: Discover the given server id for their contacts
- - PortableContact::load: Load POCO data from a given POCO address
- - check_profile: Update remote profile data
- */
-
- $search = "";
- $mode = 0;
- if ($command == "dirsearch") {
- $search = urldecode($param1);
- $mode = 1;
- } elseif ($command == "checkcontact") {
- $mode = 2;
- } elseif ($command == "suggestions") {
- $mode = 3;
- } elseif ($command == "server") {
- $mode = 4;
- } elseif ($command == "update_server") {
- $mode = 5;
- } elseif ($command == "update_server_directory") {
- $mode = 6;
- } elseif ($command == "load") {
- $mode = 7;
- } elseif ($command == "check_profile") {
- $mode = 8;
- } elseif ($command !== "") {
- Logger::log("Unknown or missing parameter ".$command."\n");
- return;
- }
-
- Logger::log('start '.$search);
-
- if ($mode == 8) {
- if ($param1 != "") {
- GContact::updateFromProbe($param1, true);
- }
- } elseif ($mode == 7) {
- if (!empty($param4)) {
- $url = $param4;
- } else {
- $url = '';
- }
- PortableContact::load(intval($param1), intval($param2), intval($param3), $url);
- } elseif ($mode == 6) {
- PortableContact::discoverSingleServer(intval($param1));
- } elseif ($mode == 5) {
- self::updateServer();
- } elseif ($mode == 4) {
- $server_url = $param1;
- if ($server_url == "") {
- return;
- }
- $server_url = filter_var($server_url, FILTER_SANITIZE_URL);
- if (substr(Strings::normaliseLink($server_url), 0, 7) != "http://") {
- return;
- }
- $result = "Checking server ".$server_url." - ";
- $ret = GServer::check($server_url);
- if ($ret) {
- $result .= "success";
- } else {
- $result .= "failed";
- }
- Logger::log($result, Logger::DEBUG);
- } elseif ($mode == 3) {
- GContact::updateSuggestions();
- } elseif (($mode == 2) && Config::get('system', 'poco_completion')) {
- self::discoverUsers();
- } elseif (($mode == 1) && ($search != "") && Config::get('system', 'poco_local_search')) {
- self::discoverDirectory($search);
- self::gsSearchUser($search);
- } elseif (($mode == 0) && ($search == "") && (Config::get('system', 'poco_discovery') != PortableContact::DISABLED)) {
- // Query Friendica and Hubzilla servers for their users
- PortableContact::discover();
-
- // Query GNU Social servers for their users ("statistics" addon has to be enabled on the GS server)
- if (!Config::get('system', 'ostatus_disabled')) {
- GContact::discoverGsUsers();
- }
- }
-
- Logger::log('end '.$search);
-
- return;
- }
-
- /**
- * @brief Updates the first 250 servers
- *
- */
- private static function updateServer() {
- $r = q("SELECT `url`, `created`, `last_failure`, `last_contact` FROM `gserver` ORDER BY rand()");
-
- if (!DBA::isResult($r)) {
- return;
- }
-
- $updated = 0;
-
- foreach ($r AS $server) {
- if (!PortableContact::updateNeeded($server["created"], "", $server["last_failure"], $server["last_contact"])) {
- continue;
- }
- Logger::log('Update server status for server '.$server["url"], Logger::DEBUG);
-
- Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $server["url"]);
-
- if (++$updated > 250) {
- return;
- }
- }
- }
-
- private static function discoverUsers() {
- Logger::log("Discover users", Logger::DEBUG);
-
- $starttime = time();
-
- $users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url`, `network` FROM `gcontact`
- WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
- `last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
- `network` IN ('%s', '%s', '%s', '%s', '') ORDER BY rand()",
- DBA::escape(Protocol::DFRN), DBA::escape(Protocol::DIASPORA),
- DBA::escape(Protocol::OSTATUS), DBA::escape(Protocol::FEED));
-
- if (!$users) {
- return;
- }
- $checked = 0;
-
- foreach ($users AS $user) {
-
- $urlparts = parse_url($user["url"]);
- if (!isset($urlparts["scheme"])) {
- DBA::update('gcontact', ['network' => Protocol::PHANTOM],
- ['nurl' => Strings::normaliseLink($user["url"])]);
- continue;
- }
-
- if (in_array($urlparts["host"], ["twitter.com", "identi.ca"])) {
- $networks = ["twitter.com" => Protocol::TWITTER, "identi.ca" => Protocol::PUMPIO];
-
- DBA::update('gcontact', ['network' => $networks[$urlparts["host"]]],
- ['nurl' => Strings::normaliseLink($user["url"])]);
- continue;
- }
-
- $server_url = Contact::getBasepath($user["url"]);
- $force_update = false;
-
- if ($user["server_url"] != "") {
-
- $force_update = (Strings::normaliseLink($user["server_url"]) != Strings::normaliseLink($server_url));
-
- $server_url = $user["server_url"];
- }
-
- if ((($server_url == "") && ($user["network"] == Protocol::FEED)) || $force_update || GServer::check($server_url, $user["network"])) {
- Logger::log('Check profile '.$user["url"]);
- Worker::add(PRIORITY_LOW, "DiscoverPoCo", "check_profile", $user["url"]);
-
- if (++$checked > 100) {
- return;
- }
- } else {
- DBA::update('gcontact', ['last_failure' => DateTimeFormat::utcNow()],
- ['nurl' => Strings::normaliseLink($user["url"])]);
- }
-
- // Quit the loop after 3 minutes
- if (time() > ($starttime + 180)) {
- return;
- }
- }
- }
-
- private static function discoverDirectory($search) {
-
- $data = Cache::get("dirsearch:".$search);
- if (!is_null($data)) {
- // Only search for the same item every 24 hours
- if (time() < $data + (60 * 60 * 24)) {
- Logger::log("Already searched for ".$search." in the last 24 hours", Logger::DEBUG);
- return;
- }
- }
-
- $x = Network::fetchUrl(get_server()."/lsearch?p=1&n=500&search=".urlencode($search));
- $j = json_decode($x);
-
- if (!empty($j->results)) {
- foreach ($j->results as $jj) {
- // Check if the contact already exists
- $exists = q("SELECT `id`, `last_contact`, `last_failure`, `updated` FROM `gcontact` WHERE `nurl` = '%s'", Strings::normaliseLink($jj->url));
- if (DBA::isResult($exists)) {
- Logger::log("Profile ".$jj->url." already exists (".$search.")", Logger::DEBUG);
-
- if (($exists[0]["last_contact"] < $exists[0]["last_failure"]) &&
- ($exists[0]["updated"] < $exists[0]["last_failure"])) {
- continue;
- }
- // Update the contact
- GContact::updateFromProbe($jj->url);
- continue;
- }
-
- $server_url = Contact::getBasepath($jj->url);
- if ($server_url != '') {
- if (!GServer::check($server_url)) {
- Logger::log("Friendica server ".$server_url." doesn't answer.", Logger::DEBUG);
- continue;
- }
- Logger::log("Friendica server ".$server_url." seems to be okay.", Logger::DEBUG);
- }
-
- $data = Probe::uri($jj->url);
- if ($data["network"] == Protocol::DFRN) {
- Logger::log("Profile ".$jj->url." is reachable (".$search.")", Logger::DEBUG);
- Logger::log("Add profile ".$jj->url." to local directory (".$search.")", Logger::DEBUG);
-
- if ($jj->tags != "") {
- $data["keywords"] = $jj->tags;
- }
-
- $data["server_url"] = $data["baseurl"];
-
- GContact::update($data);
- } else {
- Logger::log("Profile ".$jj->url." is not responding or no Friendica contact - but network ".$data["network"], Logger::DEBUG);
- }
- }
- }
- Cache::set("dirsearch:".$search, time(), Cache::DAY);
- }
-
- /**
- * @brief Search for GNU Social user with gstools.org
- *
- * @param string $search User name
- * @return bool
- * @throws \Friendica\Network\HTTPException\InternalServerErrorException
- * @throws \ImagickException
- */
- private static function gsSearchUser($search) {
-
- // Currently disabled, since the service isn't available anymore.
- // It is not removed since I hope that there will be a successor.
- return false;
-
- $url = "http://gstools.org/api/users_search/".urlencode($search);
-
- $curlResult = Network::curl($url);
- if (!$curlResult->isSuccess()) {
- return false;
- }
-
- $contacts = json_decode($curlResult->getBody());
-
- if ($contacts->status == 'ERROR') {
- return false;
- }
-
- /// @TODO AS is considered as a notation for constants (as they usually being written all upper-case)
- /// @TODO find all those and convert to all lower-case which is a keyword then
- foreach ($contacts->data AS $user) {
- $contact = Probe::uri($user->site_address."/".$user->name);
- if ($contact["network"] != Protocol::PHANTOM) {
- $contact["about"] = $user->description;
- GContact::update($contact);
- }
- }
- }
-}
--- /dev/null
+<?php
+/**
+ * @file src/Worker/FetchPoCo.php
+ */
+namespace Friendica\Worker;
+
+use Friendica\Core\Logger;
+use Friendica\Protocol\PortableContact;
+
+class FetchPoCo
+{
+ /**
+ * Fetch PortableContacts from a given PoCo server address
+ *
+ * @param integer $cid Contact ID
+ * @param integer $uid User ID
+ * @param integer $zcid Global Contact ID
+ * @param integer $url PoCo address that should be polled
+ */
+ public static function execute($cid, $uid, $zcid, $url)
+ {
+ PortableContact::load($cid, $uid, $zcid, $url);
+ }
+}
+++ /dev/null
-<?php
-/**
- * @file src/Worker/GProbe.php
- */
-
-namespace Friendica\Worker;
-
-use Friendica\Core\Cache;
-use Friendica\Core\Logger;
-use Friendica\Core\Protocol;
-use Friendica\Database\DBA;
-use Friendica\Model\GContact;
-use Friendica\Network\Probe;
-use Friendica\Protocol\PortableContact;
-use Friendica\Util\Strings;
-
-class GProbe {
- public static function execute($url = '')
- {
- if (empty($url)) {
- return;
- }
-
- $r = q(
- "SELECT `id`, `url`, `network` FROM `gcontact` WHERE `nurl` = '%s' ORDER BY `id` LIMIT 1",
- DBA::escape(Strings::normaliseLink($url))
- );
-
- Logger::log("gprobe start for ".Strings::normaliseLink($url), Logger::DEBUG);
-
- if (!DBA::isResult($r)) {
- // Is it a DDoS attempt?
- $urlparts = parse_url($url);
-
- $result = Cache::get("gprobe:".$urlparts["host"]);
- if (!is_null($result)) {
- if (in_array($result["network"], [Protocol::FEED, Protocol::PHANTOM])) {
- Logger::debug("DDoS attempt detected for " . $urlparts["host"] . " by " . ($_SERVER["REMOTE_ADDR"] ?? ''), ['$_SERVER' => $_SERVER]);
- return;
- }
- }
-
- $arr = Probe::uri($url);
-
- if (is_null($result)) {
- Cache::set("gprobe:".$urlparts["host"], $arr);
- }
-
- if (!in_array($arr["network"], [Protocol::FEED, Protocol::PHANTOM])) {
- GContact::update($arr);
- }
-
- $r = q(
- "SELECT `id`, `url`, `network` FROM `gcontact` WHERE `nurl` = '%s' ORDER BY `id` LIMIT 1",
- DBA::escape(Strings::normaliseLink($url))
- );
- }
- if (DBA::isResult($r)) {
- // Check for accessibility and do a poco discovery
- if (GContact::updateFromProbe($r[0]['url'], true) && ($r[0]["network"] == Protocol::DFRN)) {
- PortableContact::loadWorker(0, 0, $r[0]['id'], str_replace('/profile/', '/poco/', $r[0]['url']));
- }
- }
-
- Logger::log("gprobe end for ".Strings::normaliseLink($url), Logger::DEBUG);
- return;
- }
-}
--- /dev/null
+<?php
+/**
+ * @file src/Worker/SearchDirectory.php
+ */
+namespace Friendica\Worker;
+
+use Friendica\Core\Cache;
+use Friendica\Core\Config;
+use Friendica\Core\Logger;
+use Friendica\Core\Protocol;
+use Friendica\Database\DBA;
+use Friendica\Model\GContact;
+use Friendica\Model\Contact;
+use Friendica\Model\GServer;
+use Friendica\Network\Probe;
+use Friendica\Util\Network;
+use Friendica\Util\Strings;
+
+class SearchDirectory
+{
+ // <search pattern>: Searches for "search pattern" in the directory.
+ public static function execute($search)
+ {
+ if (!Config::get('system', 'poco_local_search')) {
+ Logger::info('Local search is not enabled');
+ return;
+ }
+
+ $data = Cache::get('SearchDirectory:' . $search);
+ if (!is_null($data)) {
+ // Only search for the same item every 24 hours
+ if (time() < $data + (60 * 60 * 24)) {
+ Logger::info('Already searched this in the last 24 hours', ['search' => $search]);
+ return;
+ }
+ }
+
+ $x = Network::fetchUrl(get_server() . '/lsearch?p=1&n=500&search=' . urlencode($search));
+ $j = json_decode($x);
+
+ if (!empty($j->results)) {
+ foreach ($j->results as $jj) {
+ // Check if the contact already exists
+ $gcontact = DBA::selectFirst('gcontact', ['id', 'last_contact', 'last_failure', 'updated'], ['nurl' => Strings::normaliseLink($jj->url)]);
+ if (DBA::isResult($gcontact)) {
+ Logger::info('Profile already exists', ['profile' => $jj->url, 'search' => $search]);
+
+ if (($gcontact['last_contact'] < $gcontact['last_failure']) &&
+ ($gcontact['updated'] < $gcontact['last_failure'])) {
+ continue;
+ }
+
+ // Update the contact
+ GContact::updateFromProbe($jj->url);
+ continue;
+ }
+
+ $server_url = Contact::getBasepath($jj->url);
+ if ($server_url != '') {
+ if (!GServer::check($server_url)) {
+ Logger::info("Friendica server doesn't answer.", ['server' => $server_url]);
+ continue;
+ }
+ Logger::info('Friendica server seems to be okay.', ['server' => $server_url]);
+ }
+
+ $data = Probe::uri($jj->url);
+ if ($data['network'] == Protocol::DFRN) {
+ Logger::info('Add profile to local directory', ['profile' => $jj->url]);
+
+ if ($jj->tags != '') {
+ $data['keywords'] = $jj->tags;
+ }
+
+ $data['server_url'] = $data['baseurl'];
+
+ GContact::update($data);
+ } else {
+ Logger::info('Profile is not responding or no Friendica contact', ['profile' => $jj->url, 'network' => $data['network']]);
+ }
+ }
+ }
+ Cache::set('SearchDirectory:' . $search, time(), Cache::DAY);
+ }
+}
class UpdateContact
{
+ /**
+ * Update contact data via probe
+ * @param int $contact_id Contact ID
+ * @param string $command
+ */
public static function execute($contact_id, $command = '')
{
$force = ($command == "force");
class UpdateGContact
{
+ /**
+ * Update global contact via probe
+ * @param string $url Global contact url
+ * @param string $command
+ */
public static function execute($url, $command = '')
{
$force = ($command == "force");
--- /dev/null
+<?php
+/**
+ * @file src/Worker/UpdateGContacts.php
+ */
+namespace Friendica\Worker;
+
+use Friendica\Core\Config;
+use Friendica\Core\Logger;
+use Friendica\Core\Protocol;
+use Friendica\Core\Worker;
+use Friendica\Database\DBA;
+use Friendica\Model\Contact;
+use Friendica\Model\GServer;
+use Friendica\Util\DateTimeFormat;
+use Friendica\Util\Strings;
+
+class UpdateGContacts
+{
+ /**
+ * Updates global contacts
+ */
+ public static function execute()
+ {
+ if (!Config::get('system', 'poco_completion')) {
+ return;
+ }
+
+ Logger::info('Update global contacts');
+
+ $starttime = time();
+
+ $contacts = DBA::p("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url`, `network` FROM `gcontact`
+ WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
+ `last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
+ `network` IN (?, ?, ?, ?, ?, '') ORDER BY rand()",
+ Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, Protocol::FEED);
+
+ $checked = 0;
+
+ while ($contact = DBA::fetch($contacts)) {
+ $urlparts = parse_url($contact['url']);
+ if (empty($urlparts['scheme'])) {
+ DBA::update('gcontact', ['network' => Protocol::PHANTOM],
+ ['nurl' => Strings::normaliseLink($contact['url'])]);
+ continue;
+ }
+
+ if (in_array($urlparts['host'], ['twitter.com', 'identi.ca'])) {
+ $networks = ['twitter.com' => Protocol::TWITTER, 'identi.ca' => Protocol::PUMPIO];
+
+ DBA::update('gcontact', ['network' => $networks[$urlparts['host']]],
+ ['nurl' => Strings::normaliseLink($contact['url'])]);
+ continue;
+ }
+
+ $server_url = Contact::getBasepath($contact['url']);
+ $force_update = false;
+
+ if (!empty($contact['server_url'])) {
+ $force_update = (Strings::normaliseLink($contact['server_url']) != Strings::normaliseLink($server_url));
+
+ $server_url = $contact['server_url'];
+ }
+
+ if ((empty($server_url) && ($contact['network'] == Protocol::FEED)) || $force_update || GServer::check($server_url, $contact['network'])) {
+ Logger::info('Check profile', ['profile' => $contact['url']]);
+ Worker::add(PRIORITY_LOW, 'UpdateGContact', $contact['url'], 'force');
+
+ if (++$checked > 100) {
+ return;
+ }
+ } else {
+ DBA::update('gcontact', ['last_failure' => DateTimeFormat::utcNow()],
+ ['nurl' => Strings::normaliseLink($contact['url'])]);
+ }
+
+ // Quit the loop after 3 minutes
+ if (time() > ($starttime + 180)) {
+ return;
+ }
+ }
+ }
+}
--- /dev/null
+<?php
+/**
+ * @file src/Worker/UpdateGServer.php
+ */
+namespace Friendica\Worker;
+
+use Friendica\Core\Logger;
+use Friendica\Model\GServer;
+use Friendica\Util\Strings;
+
+class UpdateGServer
+{
+ /**
+ * Update the given server
+ * @param string $server_url Server URL
+ */
+ public static function execute($server_url)
+ {
+ if (empty($server_url)) {
+ return;
+ }
+
+ $server_url = filter_var($server_url, FILTER_SANITIZE_URL);
+ if (substr(Strings::normaliseLink($server_url), 0, 7) != 'http://') {
+ return;
+ }
+
+ $ret = GServer::check($server_url);
+ Logger::info('Updated gserver', ['url' => $server_url, 'result' => $ret]);
+ }
+}
--- /dev/null
+<?php
+/**
+ * @file src/Worker/UpdateGServers.php
+ */
+namespace Friendica\Worker;
+
+use Friendica\Core\Logger;
+use Friendica\Core\Worker;
+use Friendica\Database\DBA;
+use Friendica\Model\GServer;
+
+class UpdateGServers
+{
+ /**
+ * Updates the first 250 servers
+ */
+ public static function execute()
+ {
+ $gservers = DBA::p("SELECT `url`, `created`, `last_failure`, `last_contact` FROM `gserver` ORDER BY rand()");
+ if (!DBA::isResult($gservers)) {
+ return;
+ }
+
+ $updated = 0;
+
+ while ($gserver = DBA::fetch($gservers)) {
+ if (!GServer::updateNeeded($gserver['created'], '', $gserver['last_failure'], $gserver['last_contact'])) {
+ continue;
+ }
+ Logger::info('Update server status', ['server' => $gserver['url']]);
+
+ Worker::add(PRIORITY_LOW, 'UpdateGServer', $gserver['url']);
+
+ if (++$updated > 250) {
+ return;
+ }
+ }
+ }
+}
--- /dev/null
+<?php
+/**
+ * @file src/Worker/UpdateServerDirectories.php
+ */
+namespace Friendica\Worker;
+
+use Friendica\Core\Config;
+use Friendica\Core\Logger;
+use Friendica\Model\GContact;
+use Friendica\Protocol\PortableContact;
+
+class UpdateServerDirectories
+{
+ /**
+ * Query global servers for their users
+ */
+ public static function execute()
+ {
+ if (Config::get('system', 'poco_discovery') == PortableContact::DISABLED) {
+ return;
+ }
+
+ // Query Friendica and Hubzilla servers for their users
+ PortableContact::discover();
+
+ // Query GNU Social servers for their users ("statistics" addon has to be enabled on the GS server)
+ if (!Config::get('system', 'ostatus_disabled')) {
+ GContact::discoverGsUsers();
+ }
+ }
+}
--- /dev/null
+<?php
+/**
+ * @file src/Worker/UpdateServerDirectory.php
+ */
+namespace Friendica\Worker;
+
+use Friendica\Core\Logger;
+use Friendica\Model\GServer;
+
+class UpdateServerDirectory
+{
+ /**
+ * Query the given server for their users
+ * @param string $gserver Server URL
+ */
+ public static function execute($gserver)
+ {
+ GServer::updateDirectory($gserver);
+ return;
+ }
+}
--- /dev/null
+<?php
+/**
+ * @file src/Worker/UpdateSuggestions.php
+ */
+namespace Friendica\Worker;
+
+use Friendica\Core\Logger;
+use Friendica\Model\GContact;
+
+class UpdateSuggestions
+{
+ /**
+ * Discover other servers for their contacts.
+ */
+ public static function execute()
+ {
+ GContact::updateSuggestions();
+ }
+}
use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) {
- define('DB_UPDATE_VERSION', 1327);
+ define('DB_UPDATE_VERSION', 1328);
}
return [
"info" => ["type" => "text", "comment" => ""],
"register_policy" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => ""],
"registered-users" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "comment" => "Number of registered users"],
+ "directory-type" => ["type" => "tinyint", "default" => "0", "comment" => "Type of directory service (Poco, Mastodon)"],
"poco" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"noscrape" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""],
projects / friendica.git / commitdiff