Merge pull request #12557 from annando/reduce-update-contacts

Reduce the amount of "UpdateContact" worker calls

diff --git a/src/Model/Contact.php b/src/Model/Contact.php
index 2003b4608e84c88eddf36f489a8fda826a74a1c7..e928df7f6323570bd399d273c466acefab4316a5 100644 (file)
--- a/src/Model/Contact.php
+++ b/src/Model/Contact.php
@@ -329,7 +329,7 @@ class Contact
                // Add internal fields
                $removal = [];
                if (!empty($fields)) {
-                       foreach (['id', 'next-update', 'network'] as $internal) {
+                       foreach (['id', 'next-update', 'network', 'local-data'] as $internal) {
                                if (!in_array($internal, $fields)) {
                                        $fields[] = $internal;
                                        $removal[] = $internal;
@@ -358,8 +358,10 @@ class Contact
                        return [];
                }
 
+               $background_update = DI::config()->get('system', 'update_active_contacts') ? $contact['local-data'] : true;
+
                // Update the contact in the background if needed
-               if (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
+               if ($background_update && !self::isLocal($url) && Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
                        Worker::add(['priority' => Worker::PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']);
                }
 
@@ -1266,12 +1268,14 @@ class Contact
                        return 0;
                }
 
-               $contact = self::getByURL($url, false, ['id', 'network', 'uri-id', 'next-update'], $uid);
+               $contact = self::getByURL($url, false, ['id', 'network', 'uri-id', 'next-update', 'local-data'], $uid);
 
                if (!empty($contact)) {
                        $contact_id = $contact['id'];
 
-                       if (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
+                       $background_update = DI::config()->get('system', 'update_active_contacts') ? $contact['local-data'] : true;
+
+                       if ($background_update && !self::isLocal($url) && Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
                                Worker::add(['priority' => Worker::PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']);
                        }
 
@@ -2481,6 +2485,44 @@ class Contact
                return true;
        }
 
+       /**
+        * Perform a contact update if the contact is outdated
+        *
+        * @param integer $id contact id
+        * @return bool
+        */
+       public static function updateByIdIfNeeded(int $id): bool
+       {
+               $contact = self::selectFirst(['url'], ["`id` = ? AND `next-update` < ?", $id, DateTimeFormat::utcNow()]);
+               if (empty($contact['url'])) {
+                       return false;
+               }
+
+               if (self::isLocal($contact['url'])) {
+                       return true;
+               }
+
+               $stamp = (float)microtime(true);
+               self::updateFromProbe($id);
+               Logger::debug('Contact data is updated.', ['duration' => round((float)microtime(true) - $stamp, 3), 'id' => $id, 'url' => $contact['url'], 'callstack' => System::callstack(20)]);
+               return true;
+       }
+
+       /**
+        * Perform a contact update if the contact is outdated
+        *
+        * @param string $url contact url
+        * @return bool
+        */
+       public static function updateByUrlIfNeeded(string $url): bool
+       {
+               $id = self::getIdForURL($url, 0, false);
+               if (!empty($id)) {
+                       return self::updateByIdIfNeeded($id);
+               }
+               return (bool)self::getIdForURL($url);
+       }
+
        /**
         * Updates contact record by provided id and optional network
         *

diff --git a/src/Protocol/ActivityPub/Delivery.php b/src/Protocol/ActivityPub/Delivery.php
index add898888f9d5f994932d6691ded4841718e9f67..0e014c703563818c90d3cba0ecbbe278ac92a015 100644 (file)
--- a/src/Protocol/ActivityPub/Delivery.php
+++ b/src/Protocol/ActivityPub/Delivery.php
@@ -205,9 +205,15 @@ class Delivery
         */
        private static function setSuccess(array $receivers, bool $success)
        {
-               $gsid = null;
+               $gsid           = null;
+               $update_counter = 0;
 
                foreach ($receivers as $receiver) {
+                       // Only update the first 10 receivers to avoid flooding the remote system with requests
+                       if ($success && ($update_counter < 10) && Contact::updateByIdIfNeeded($receiver)) {
+                               $update_counter++;
+                       }
+
                        $contact = Contact::getById($receiver);
                        if (empty($contact)) {
                                continue;

diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php
index e98fa4596c122b3ac21b638a55cad6b4118a9b64..fdf35ff8685092973e99876c6a911bb9fdbdee18 100644 (file)
--- a/src/Protocol/ActivityPub/Processor.php
+++ b/src/Protocol/ActivityPub/Processor.php
@@ -1530,6 +1530,12 @@ class Processor
 
                $ldactivity['recursion-depth'] = !empty($child['recursion-depth']) ? $child['recursion-depth'] + 1 : 0;
 
+               if ($object_actor != $actor) {
+                       Contact::updateByUrlIfNeeded($object_actor);
+               }
+
+               Contact::updateByUrlIfNeeded($actor);
+
                if (!empty($relay_actor)) {
                        $ldactivity['thread-completion'] = $ldactivity['from-relay'] = Contact::getIdForURL($relay_actor);
                        $ldactivity['completion-mode']   = Receiver::COMPLETION_RELAY;

diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php
index fbe8c32dc70fe8624c6f408a0e5782aa18cfff96..3117a8791040c9650518eaa713d10badc3d0319e 100644 (file)
--- a/src/Protocol/ActivityPub/Receiver.php
+++ b/src/Protocol/ActivityPub/Receiver.php
@@ -315,12 +315,15 @@ class Receiver
                        $object_type = JsonLD::fetchElement($activity['as:object'], '@type');
                }
 
+               $fetched = false;
+
                if (!empty($id) && !$trust_source) {
                        $fetch_uid = $uid ?: self::getBestUserForActivity($activity);
 
                        $fetched_activity = Processor::fetchCachedActivity($fetch_id, $fetch_uid);
                        if (!empty($fetched_activity)) {
-                               $object = JsonLD::compact($fetched_activity);
+                               $fetched = true;
+                               $object  = JsonLD::compact($fetched_activity);
 
                                $fetched_id   = JsonLD::fetchElement($object, '@id');
                                $fetched_type = JsonLD::fetchElement($object, '@type');
@@ -351,7 +354,7 @@ class Receiver
                $type = JsonLD::fetchElement($activity, '@type');
 
                // Fetch all receivers from to, cc, bto and bcc
-               $receiverdata = self::getReceivers($activity, $actor);
+               $receiverdata = self::getReceivers($activity, $actor, [], false, $push || $fetched);
                $receivers = $reception_types = [];
                foreach ($receiverdata as $key => $data) {
                        $receivers[$key] = $data['uid'];
@@ -1029,7 +1032,7 @@ class Receiver
                $uid = 0;
                $actor = JsonLD::fetchElement($activity, 'as:actor', '@id') ?? '';
 
-               $receivers = self::getReceivers($activity, $actor);
+               $receivers = self::getReceivers($activity, $actor, [], false, false);
                foreach ($receivers as $receiver) {
                        if ($receiver['type'] == self::TARGET_GLOBAL) {
                                return 0;
@@ -1076,14 +1079,15 @@ class Receiver
         * Fetch the receiver list from an activity array
         *
         * @param array   $activity
-        * @param string  $actor
-        * @param array   $tags
-        * @param boolean $fetch_unlisted
+        * @param string $actor
+        * @param array  $tags
+        * @param bool   $fetch_unlisted
+        * @param bool   $push
         *
         * @return array with receivers (user id)
         * @throws \Exception
         */
-       private static function getReceivers(array $activity, string $actor, array $tags = [], bool $fetch_unlisted = false): array
+       private static function getReceivers(array $activity, string $actor, array $tags, bool $fetch_unlisted, bool $push): array
        {
                $reply = $receivers = $profile = [];
 
@@ -1117,6 +1121,9 @@ class Receiver
                        $profile   = APContact::getByURL($actor);
                        $followers = $profile['followers'] ?? '';
                        $is_forum  = ($actor['type'] ?? '') == 'Group';
+                       if ($push) {
+                               Contact::updateByUrlIfNeeded($actor);
+                       }
                        Logger::info('Got actor and followers', ['actor' => $actor, 'followers' => $followers]);
                } else {
                        Logger::info('Empty actor', ['activity' => $activity]);
@@ -2006,7 +2013,7 @@ class Receiver
                        $object_data['question'] = self::processQuestion($object);
                }
 
-               $receiverdata = self::getReceivers($object, $object_data['actor'] ?? '', $object_data['tags'], true);
+               $receiverdata = self::getReceivers($object, $object_data['actor'] ?? '', $object_data['tags'], true, false);
                $receivers = $reception_types = [];
                foreach ($receiverdata as $key => $data) {
                        $receivers[$key] = $data['uid'];

diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php
index 86d88867b922397c415581df40312a7f2bac52e1..0cb2a66cf17f528ec1735db03ae385cfa6527720 100644 (file)
--- a/src/Protocol/Diaspora.php
+++ b/src/Protocol/Diaspora.php
@@ -534,7 +534,7 @@ class Diaspora
                if (is_null($fields)) {
                        $private = true;
                        if (!($fields = self::validPosting($msg))) {
-                               Logger::warning('Invalid posting', ['msg' => $msg]);
+                               Logger::notice('Invalid posting', ['msg' => $msg]);
                                return false;
                        }
                } else {
@@ -812,6 +812,7 @@ class Diaspora
         */
        private static function contactByHandle(int $uid, WebFingerUri $uri): array
        {
+               Contact::updateByUrlIfNeeded($uri->getAddr());
                return Contact::getByURL($uri->getAddr(), null, [], $uid);
        }
 

diff --git a/src/Worker/UpdateContacts.php b/src/Worker/UpdateContacts.php
index ed4a9932f134aa3ed688e4292a4ebff809da73ec..881418dc11f6f3cfed82add4c1188d199ab0cf8d 100644 (file)
--- a/src/Worker/UpdateContacts.php
+++ b/src/Worker/UpdateContacts.php
@@ -22,10 +22,11 @@
 namespace Friendica\Worker;
 
 use Friendica\Core\Logger;
-use Friendica\Core\Protocol;
 use Friendica\Core\Worker;
 use Friendica\Database\DBA;
 use Friendica\DI;
+use Friendica\Model\Contact;
+use Friendica\Model\GServer;
 use Friendica\Util\DateTimeFormat;
 
 /**
@@ -56,10 +57,19 @@ class UpdateContacts
                }
 
                $condition = DBA::mergeConditions(["`next-update` < ?", DateTimeFormat::utcNow()], $condition);
-               $contacts = DBA::select('contact', ['id'], $condition, ['order' => ['next-update'], 'limit' => $limit]);
+               $contacts = DBA::select('contact', ['id', 'url', 'gsid', 'baseurl'], $condition, ['order' => ['next-update'], 'limit' => $limit]);
                $count = 0;
                while ($contact = DBA::fetch($contacts)) {
-                       if (Worker::add(['priority' => Worker::PRIORITY_LOW, 'dont_fork' => true], "UpdateContact", $contact['id'])) {
+                       if (Contact::isLocal($contact['url'])) {
+                               continue;
+                       }
+                       if ((!empty($contact['gsid']) || !empty($contact['baseurl'])) && GServer::reachable($contact)) {
+                               $stamp = (float)microtime(true);
+                               $success = Contact::updateFromProbe($contact['id']);
+                               Logger::debug('Direct update', ['id' => $contact['id'], 'count' => $count, 'duration' => round((float)microtime(true) - $stamp, 3), 'success' => $success]);
+                               ++$count;
+                       } elseif (Worker::add(['priority' => Worker::PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id'])) {
+                               Logger::debug('Update by worker', ['id' => $contact['id'], 'count' => $count]);
                                ++$count;
                        }
                        Worker::coolDown();