The fcontact table is now updated in the background to improve performance

diff --git a/database.sql b/database.sql
index b76a4e0206005444354908fe7b2a978652fc39ae..010b64542a8e276ff95d1224f3dd0524077740dc 100644 (file)
--- a/database.sql
+++ b/database.sql
@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2022.09-rc (Giant Rhubarb)
--- DB_UPDATE_VERSION 1482
+-- DB_UPDATE_VERSION 1483
 -- ------------------------------------------
 
 
@@ -625,6 +625,7 @@ CREATE TABLE IF NOT EXISTS `fcontact` (
        `network` char(4) NOT NULL DEFAULT '' COMMENT '',
        `alias` varbinary(383) NOT NULL DEFAULT '' COMMENT '',
        `pubkey` text COMMENT '',
+       `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
        `updated` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
        `interacting_count` int unsigned DEFAULT 0 COMMENT 'Number of contacts this contact interactes with',
        `interacted_count` int unsigned DEFAULT 0 COMMENT 'Number of contacts that interacted with this contact',

diff --git a/doc/database/db_fcontact.md b/doc/database/db_fcontact.md
index d082b1717462b4002278ee6f374d4a064aba716b..095c47aafd467d514c678837ac19fa0d1c31f20a 100644 (file)
--- a/doc/database/db_fcontact.md
+++ b/doc/database/db_fcontact.md
@@ -25,6 +25,7 @@ Fields
 | network           |                                                               | char(4)          | NO   |     |                     |                |
 | alias             |                                                               | varbinary(383)   | NO   |     |                     |                |
 | pubkey            |                                                               | text             | YES  |     | NULL                |                |
+| created           |                                                               | datetime         | NO   |     | 0001-01-01 00:00:00 |                |
 | updated           |                                                               | datetime         | NO   |     | 0001-01-01 00:00:00 |                |
 | interacting_count | Number of contacts this contact interactes with               | int unsigned     | YES  |     | 0                   |                |
 | interacted_count  | Number of contacts that interacted with this contact          | int unsigned     | YES  |     | 0                   |                |

diff --git a/src/Model/Contact.php b/src/Model/Contact.php
index 6a57693991127be745e836acf043a08ecec04ed9..1f5dbcdcab17e139cb15b4e1480456ebedf9a0ed 100644 (file)
--- a/src/Model/Contact.php
+++ b/src/Model/Contact.php
@@ -252,7 +252,7 @@ class Contact
                // Add internal fields
                $removal = [];
                if (!empty($fields)) {
-                       foreach (['id', 'avatar', 'created', 'updated', 'last-update', 'success_update', 'failure_update', 'network'] as $internal) {
+                       foreach (['id', 'next-update', 'network'] as $internal) {
                                if (!in_array($internal, $fields)) {
                                        $fields[] = $internal;
                                        $removal[] = $internal;
@@ -282,9 +282,8 @@ class Contact
                }
 
                // Update the contact in the background if needed
-               $updated = max($contact['success_update'], $contact['created'], $contact['updated'], $contact['last-update'], $contact['failure_update']);
-               if (($updated < DateTimeFormat::utc('now -7 days')) && in_array($contact['network'], Protocol::FEDERATED) && !self::isLocalById($contact['id'])) {
-                       Worker::add(PRIORITY_LOW, "UpdateContact", $contact['id']);
+               if (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
+                       Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']);
                }
 
                // Remove the internal fields
@@ -935,7 +934,6 @@ class Contact
 
                DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_CONTACTS . 'followers:' . $uid);
                DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_CONTACTS . 'following:' . $uid);
-               DI::cache()->delete(NoScrape::CACHEKEY . $uid);
        }
 
        /**
@@ -1194,11 +1192,15 @@ class Contact
                        return 0;
                }
 
-               $contact = self::getByURL($url, false, ['id', 'network', 'uri-id'], $uid);
+               $contact = self::getByURL($url, false, ['id', 'network', 'uri-id', 'next-update'], $uid);
 
                if (!empty($contact)) {
                        $contact_id = $contact['id'];
 
+                       if (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
+                               Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']);
+                       }
+
                        if (empty($update) && (!empty($contact['uri-id']) || is_bool($update))) {
                                Logger::debug('Contact found', ['url' => $url, 'uid' => $uid, 'update' => $update, 'cid' => $contact_id]);
                                return $contact_id;
@@ -2460,7 +2462,7 @@ class Contact
 
                $has_local_data = self::hasLocalData($id, $contact);
 
-               if (!in_array($ret['network'], array_merge(Protocol::FEDERATED, [Protocol::ZOT, Protocol::PHANTOM]))) {
+               if (!Probe::isProbable($ret['network'])) {
                        // Periodical checks are only done on federated contacts
                        $failed_next_update  = null;
                        $success_next_update = null;
@@ -3320,12 +3322,12 @@ class Contact
                        if (empty($url) || !is_string($url)) {
                                continue;
                        }
-                       $contact = self::getByURL($url, false, ['id', 'updated']);
+                       $contact = self::getByURL($url, false, ['id', 'network', 'next-update']);
                        if (empty($contact['id']) && Network::isValidHttpUrl($url)) {
                                Worker::add(PRIORITY_LOW, 'AddContact', 0, $url);
                                ++$added;
-                       } elseif ($contact['updated'] < DateTimeFormat::utc('now -7 days')) {
-                               Worker::add(PRIORITY_LOW, 'UpdateContact', $contact['id']);
+                       } elseif (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
+                               Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']);
                                ++$updated;
                        } else {
                                ++$unchanged;

diff --git a/src/Model/FContact.php b/src/Model/FContact.php
index 62f2f983648e824e33e12958a0eda4fa4c592b8e..ae6c9cb316a76c7426bcf513aa5d9cbc60c23f2f 100644 (file)
--- a/src/Model/FContact.php
+++ b/src/Model/FContact.php
@@ -23,6 +23,7 @@ namespace Friendica\Model;
 
 use Friendica\Core\Logger;
 use Friendica\Core\Protocol;
+use Friendica\Core\Worker;
 use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Network\Probe;
@@ -43,6 +44,7 @@ class FContact
         */
        public static function getByURL(string $handle, $update = null): array
        {
+               Logger::debug('Fetch fcontact', ['handle' => $handle, 'update' => $update]);
                $person = DBA::selectFirst('fcontact', [], ['network' => Protocol::DIASPORA, 'addr' => $handle]);
                if (!DBA::isResult($person)) {
                        $urls = [$handle, str_replace('http://', 'https://', $handle), Strings::normaliseLink($handle)];
@@ -50,21 +52,17 @@ class FContact
                }
 
                if (DBA::isResult($person)) {
-                       Logger::debug('In cache', ['person' => $person]);
+                       Logger::debug('In cache', ['handle' => $handle]);
 
                        if (is_null($update)) {
-                               // update record occasionally so it doesn't get stale
-                               $d = strtotime($person['updated'] . ' +00:00');
-                               if ($d < strtotime('now - 14 days')) {
-                                       $update = true;
-                               }
-
-                               if (empty($person['guid']) || empty($person['uri-id'])) {
-                                       $update = true;
+                               $update = empty($person['guid']) || empty($person['uri-id']) || ($person['created'] <= DBA::NULL_DATETIME);
+                               if (GServer::getNextUpdateDate(true, $person['created'], $person['updated'], false) < DateTimeFormat::utcNow()) {
+                                       Logger::debug('Start background update', ['handle' => $handle]);
+                                       Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateFContact', $handle);
                                }
                        }
                } elseif (is_null($update)) {
-                       $update = !DBA::isResult($person);
+                       $update = true;
                } else {
                        $person = [];
                }
@@ -95,7 +93,8 @@ class FContact
        {
                $uriid = ItemURI::insert(['uri' => $arr['url'], 'guid' => $arr['guid']]);
 
-               $contact = Contact::getByUriId($uriid, ['id']);
+               $fcontact  = DBA::selectFirst('fcontact', ['created'], ['url' => $arr['url'], 'network' => $arr['network']]);
+               $contact   = Contact::getByUriId($uriid, ['id', 'created']);
                $apcontact = APContact::getByURL($arr['url'], false);
                if (!empty($apcontact)) {
                        $interacted  = $apcontact['following_count'];
@@ -129,10 +128,14 @@ class FContact
                        'updated' => DateTimeFormat::utcNow(),
                ];
 
-               $condition = ['url' => $arr['url'], 'network' => $arr['network']];
+               if (empty($fcontact['created'])) {
+                       $fields['created'] = $fields['updated'];
+               } elseif (!empty($contact['created']) && ($fcontact['created'] <= DBA::NULL_DATETIME)) {
+                       $fields['created'] = $contact['created'];
+               }
 
                $fields = DI::dbaDefinition()->truncateFieldsForTable('fcontact', $fields);
-               DBA::update('fcontact', $fields, $condition, true);
+               DBA::update('fcontact', $fields, ['url' => $arr['url'], 'network' => $arr['network']], true);
        }
 
        /**

diff --git a/src/Module/NoScrape.php b/src/Module/NoScrape.php
index e3a01ceaab6d29467d47ceda5c4a0afc1fad9e55..363acd38c2decdc7296ac05c391473428d3bebd8 100644 (file)
--- a/src/Module/NoScrape.php
+++ b/src/Module/NoScrape.php
@@ -22,11 +22,10 @@
 namespace Friendica\Module;
 
 use Friendica\BaseModule;
-use Friendica\Core\Cache\Enum\Duration;
-use Friendica\Core\Protocol;
 use Friendica\Core\System;
 use Friendica\Database\DBA;
 use Friendica\DI;
+use Friendica\Model\APContact;
 use Friendica\Model\User;
 
 /**
@@ -36,8 +35,6 @@ use Friendica\Model\User;
  */
 class NoScrape extends BaseModule
 {
-       const CACHEKEY = 'noscrape:';
-
        protected function rawContent(array $request = [])
        {
                $a = DI::app();
@@ -58,12 +55,6 @@ class NoScrape extends BaseModule
                        System::jsonError(404, 'Profile not found');
                }
 
-               $cachekey = self::CACHEKEY . $owner['uid'];
-               $result = DI::cache()->get($cachekey);
-               if (!is_null($result)) {
-                       System::jsonExit($result);
-               }
-
                $json_info = [
                        'addr'         => $owner['addr'],
                        'nick'         => $which,
@@ -98,16 +89,8 @@ class NoScrape extends BaseModule
                }
 
                if (!($owner['hide-friends'] ?? false)) {
-                       $json_info['contacts'] = DBA::count('contact',
-                               [
-                                       'uid'     => $owner['uid'],
-                                       'self'    => 0,
-                                       'blocked' => 0,
-                                       'pending' => 0,
-                                       'hidden'  => 0,
-                                       'archive' => 0,
-                                       'network' => [Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS]
-                               ]);
+                       $apcontact = APContact::getByURL($owner['url']);
+                       $json_info['contacts'] = max($apcontact['following_count'], $apcontact['followers_count']);
                }
 
                // We display the last activity (post or login), reduced to year and week number
@@ -135,8 +118,6 @@ class NoScrape extends BaseModule
                        }
                }
 
-               DI::cache()->set($cachekey, $json_info, Duration::DAY);
-
                System::jsonExit($json_info);
        }
 }

diff --git a/src/Network/Probe.php b/src/Network/Probe.php
index d5d87ebf955057f9cc47da804f5374f742b9c862..594ff506f23c308cf444658541c2faba34e8e00e 100644 (file)
--- a/src/Network/Probe.php
+++ b/src/Network/Probe.php
@@ -57,6 +57,18 @@ class Probe
        private static $baseurl;
        private static $istimeout;
 
+       /**
+        * Checks if the provided network can be probed
+        *
+        * @param string $network
+        * @return boolean
+        */
+       public static function isProbable(string $network): bool
+       {
+               return (in_array($network, array_merge(Protocol::FEDERATED, [Protocol::ZOT, Protocol::PHANTOM])));
+
+       }
+
        /**
         * Remove stuff from an URI that doesn't belong there
         *

diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php
index 9f1ab158589875cffac1be87f5723685cec5235d..4bba542c2467462dc5cef7531d5c25198eac40d7 100644 (file)
--- a/static/dbstructure.config.php
+++ b/static/dbstructure.config.php
@@ -55,7 +55,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-       define('DB_UPDATE_VERSION', 1482);
+       define('DB_UPDATE_VERSION', 1483);
 }
 
 return [
@@ -684,6 +684,7 @@ return [
                        "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""],
                        "alias" => ["type" => "varbinary(383)", "not null" => "1", "default" => "", "comment" => ""],
                        "pubkey" => ["type" => "text", "comment" => ""],
+                       "created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""],
                        "updated" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""],
                        "interacting_count" => ["type" => "int unsigned", "default" => 0, "comment" => "Number of contacts this contact interactes with"],
                        "interacted_count" => ["type" => "int unsigned", "default" => 0, "comment" => "Number of contacts that interacted with this contact"],