Merge pull request #8901 from annando/failed

New field in gserver, gcontact and contact for failed connections

diff --git a/database.sql b/database.sql
index a988da7bfa1e729f0ea7a6f6794be5a915c47303..deaa39a5b0535ce573afbc282b8797dec5a43730 100644 (file)
--- a/database.sql
+++ b/database.sql
@@ -1,6 +1,6 @@
 -- ------------------------------------------
--- Friendica 2020.06-dev (Red Hot Poker)
--- DB_UPDATE_VERSION 1353
+-- Friendica 2020.09-dev (Red Hot Poker)
+-- DB_UPDATE_VERSION 1357
 -- ------------------------------------------
 
 
@@ -28,6 +28,7 @@ CREATE TABLE IF NOT EXISTS `gserver` (
        `last_poco_query` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
        `last_contact` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
        `last_failure` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
+       `failed` boolean COMMENT 'Connection failed',
         PRIMARY KEY(`id`),
         UNIQUE INDEX `nurl` (`nurl`(190))
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Global servers';
@@ -95,6 +96,7 @@ CREATE TABLE IF NOT EXISTS `contact` (
        `last-update` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last try to update the contact info',
        `success_update` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last successful contact update',
        `failure_update` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last failed update',
+       `failed` boolean COMMENT 'Connection failed',
        `name-date` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
        `uri-date` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
        `avatar-date` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
@@ -137,8 +139,10 @@ CREATE TABLE IF NOT EXISTS `contact` (
         INDEX `addr_uid` (`addr`(32),`uid`),
         INDEX `nurl_uid` (`nurl`(32),`uid`),
         INDEX `nick_uid` (`nick`(32),`uid`),
+        INDEX `attag_uid` (`attag`(32),`uid`),
         INDEX `dfrn-id` (`dfrn-id`(64)),
         INDEX `issued-id` (`issued-id`(64)),
+        INDEX `network_uid_lastupdate` (`network`,`uid`,`last-update`),
         INDEX `gsid` (`gsid`),
        FOREIGN KEY (`gsid`) REFERENCES `gserver` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='contact table';
@@ -253,6 +257,7 @@ CREATE TABLE IF NOT EXISTS `apcontact` (
         INDEX `addr` (`addr`(32)),
         INDEX `alias` (`alias`(190)),
         INDEX `followers` (`followers`(190)),
+        INDEX `baseurl` (`baseurl`(190)),
         INDEX `gsid` (`gsid`),
        FOREIGN KEY (`gsid`) REFERENCES `gserver` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='ActivityPub compatible contacts - used in the ActivityPub implementation';
@@ -482,6 +487,7 @@ CREATE TABLE IF NOT EXISTS `gcontact` (
        `last_contact` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
        `last_failure` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
        `last_discovery` datetime DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last contact discovery',
+       `failed` boolean COMMENT 'Connection failed',
        `archive_date` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
        `archived` boolean NOT NULL DEFAULT '0' COMMENT '',
        `location` varchar(255) NOT NULL DEFAULT '' COMMENT '',

diff --git a/mod/poco.php b/mod/poco.php
index f1fdc55d7560bbca78b88c82443a4fcfaeaf96aa..abe10ee39cc2ef7a56540ab9008b354ae3ebf39a 100644 (file)
--- a/mod/poco.php
+++ b/mod/poco.php
@@ -110,7 +110,7 @@ function poco_init(App $a) {
                $totalResults = DBA::count('profile', ['net-publish' => true]);
        } else {
                $contacts = q("SELECT count(*) AS `total` FROM `contact` WHERE `uid` = %d AND `blocked` = 0 AND `pending` = 0 AND `hidden` = 0 AND `archive` = 0
-                       AND (`success_update` >= `failure_update` OR `last-item` >= `failure_update`)
+                       AND NOT `failed`
                        AND `network` IN ('%s', '%s', '%s', '%s') $sql_extra",
                        intval($user['uid']),
                        DBA::escape(Protocol::DFRN),
@@ -148,7 +148,7 @@ function poco_init(App $a) {
        } else {
                Logger::log("Start query for user " . $user['nickname'], Logger::DEBUG);
                $contacts = q("SELECT * FROM `contact` WHERE `uid` = %d AND `blocked` = 0 AND `pending` = 0 AND `hidden` = 0 AND `archive` = 0
-                       AND (`success_update` >= `failure_update` OR `last-item` >= `failure_update`)
+                       AND NOT `failed`
                        AND `network` IN ('%s', '%s', '%s', '%s') $sql_extra LIMIT %d, %d",
                        intval($user['uid']),
                        DBA::escape(Protocol::DFRN),

diff --git a/src/Core/Search.php b/src/Core/Search.php
index ea979610e6c20d33984bafc320f887598401f87e..edc88ffd7d4fa004c2ae8034b99d77ac5c8dafa7 100644 (file)
--- a/src/Core/Search.php
+++ b/src/Core/Search.php
@@ -180,7 +180,7 @@ class Search
                $count = DBA::count('gcontact', [
                        'NOT `hide`
                        AND `network` IN (?, ?, ?, ?)
-                       AND ((`last_contact` >= `last_failure`) OR (`updated` >= `last_failure`))
+                       AND NOT `failed`
                        AND (`url` LIKE ? OR `name` LIKE ? OR `location` LIKE ? 
                                OR `addr` LIKE ? OR `about` LIKE ? OR `keywords` LIKE ?)
                        AND `community` = ?',
@@ -199,7 +199,7 @@ class Search
                $data = DBA::select('gcontact', ['nurl', 'name', 'addr', 'url', 'photo', 'network', 'keywords'], [
                        'NOT `hide`
                        AND `network` IN (?, ?, ?, ?)
-                       AND ((`last_contact` >= `last_failure`) OR (`updated` >= `last_failure`))
+                       AND NOT `failed`
                        AND (`url` LIKE ? OR `name` LIKE ? OR `location` LIKE ? 
                                OR `addr` LIKE ? OR `about` LIKE ? OR `keywords` LIKE ?)
                        AND `community` = ?',

diff --git a/src/Model/Contact.php b/src/Model/Contact.php
index 646da57caa6170013b996fa62d81261643c69881..09d527733192565a7584d60b1a4f328b996cd662 100644 (file)
--- a/src/Model/Contact.php
+++ b/src/Model/Contact.php
@@ -1029,7 +1029,7 @@ class Contact
        {
                // Always unarchive the relay contact entry
                if (!empty($contact['batch']) && !empty($contact['term-date']) && ($contact['term-date'] > DBA::NULL_DATETIME)) {
-                       $fields = ['term-date' => DBA::NULL_DATETIME, 'archive' => false];
+                       $fields = ['failed' => false, 'term-date' => DBA::NULL_DATETIME, 'archive' => false];
                        $condition = ['uid' => 0, 'network' => Protocol::FEDERATED, 'batch' => $contact['batch'], 'contact-type' => self::TYPE_RELAY];
                        DBA::update('contact', $fields, $condition);
                }
@@ -1053,7 +1053,7 @@ class Contact
                }
 
                // It's a miracle. Our dead contact has inexplicably come back to life.
-               $fields = ['term-date' => DBA::NULL_DATETIME, 'archive' => false];
+               $fields = ['failed' => false, 'term-date' => DBA::NULL_DATETIME, 'archive' => false];
                DBA::update('contact', $fields, ['id' => $contact['id']]);
                DBA::update('contact', $fields, ['nurl' => Strings::normaliseLink($contact['url']), 'self' => false]);
                GContact::updateFromPublicContactURL($contact['url']);
@@ -1495,7 +1495,8 @@ class Contact
                        $updated = [
                                'url' => $data['url'],
                                'nurl' => Strings::normaliseLink($data['url']),
-                               'updated' => DateTimeFormat::utcNow()
+                               'updated' => DateTimeFormat::utcNow(),
+                               'failed' => false
                        ];
 
                        $fields = ['addr', 'alias', 'name', 'nick', 'keywords', 'location', 'about', 'baseurl', 'gsid'];
@@ -1973,7 +1974,7 @@ class Contact
                // We check after the probing to be able to correct falsely detected contact types.
                if (($contact['contact-type'] == self::TYPE_RELAY) &&
                        (!Strings::compareLink($ret['url'], $contact['url']) || in_array($ret['network'], [Protocol::FEED, Protocol::PHANTOM]))) {
-                       self::updateContact($id, $uid, $contact['url'], ['last-update' => $updated, 'success_update' => $updated]);
+                       self::updateContact($id, $uid, $contact['url'], ['failed' => false, 'last-update' => $updated, 'success_update' => $updated]);
                        Logger::info('Not updating relais', ['id' => $id, 'url' => $contact['url']]);
                        return true;
                }
@@ -1981,7 +1982,7 @@ class Contact
                // If Probe::uri fails the network code will be different ("feed" or "unkn")
                if (in_array($ret['network'], [Protocol::FEED, Protocol::PHANTOM]) && ($ret['network'] != $contact['network'])) {
                        if ($force && ($uid == 0)) {
-                               self::updateContact($id, $uid, $ret['url'], ['last-update' => $updated, 'failure_update' => $updated]);
+                               self::updateContact($id, $uid, $ret['url'], ['failed' => true, 'last-update' => $updated, 'failure_update' => $updated]);
                        }
                        return false;
                }
@@ -2025,7 +2026,7 @@ class Contact
 
                if (!$update) {
                        if ($force) {
-                               self::updateContact($id, $uid, $ret['url'], ['last-update' => $updated, 'success_update' => $updated]);
+                               self::updateContact($id, $uid, $ret['url'], ['failed' => false, 'last-update' => $updated, 'success_update' => $updated]);
                        }
 
                        // Update the public contact
@@ -2055,6 +2056,7 @@ class Contact
                if ($force && ($uid == 0)) {
                        $ret['last-update'] = $updated;
                        $ret['success_update'] = $updated;
+                       $ret['failed'] = false;
                }
 
                unset($ret['photo']);

diff --git a/src/Model/GContact.php b/src/Model/GContact.php
index 5ef74dcf7756d484cbc5cf0d412ebae5820bb234..6a3c7da74cbada7e4fc0f399ab22c91f497af87d 100644 (file)
--- a/src/Model/GContact.php
+++ b/src/Model/GContact.php
@@ -95,7 +95,7 @@ class GContact
 
                $results = DBA::p("SELECT `nurl` FROM `gcontact`
                        WHERE NOT `hide` AND `network` IN (?, ?, ?, ?) AND
-                               ((`last_contact` >= `last_failure`) OR (`updated` >= `last_failure`)) AND
+                               NOT `failed` AND
                                (`addr` LIKE ? OR `name` LIKE ? OR `nick` LIKE ?) $extra_sql
                                GROUP BY `nurl` ORDER BY `nurl` DESC LIMIT 1000",
                        Protocol::DFRN, Protocol::ACTIVITYPUB, $ostatus, $diaspora, $search, $search, $search
@@ -231,6 +231,7 @@ class GContact
                        }
 
                        $gcontact['server_url'] = $data['baseurl'];
+                       $gcontact['failed'] = false;
 
                        $gcontact = array_merge($gcontact, $data);
                }
@@ -273,8 +274,7 @@ class GContact
                        "SELECT count(*) as `total`
                        FROM `glink` INNER JOIN `gcontact` on `glink`.`gcid` = `gcontact`.`id`
                        WHERE `glink`.`cid` = %d AND `glink`.`uid` = %d AND
-                       ((`gcontact`.`last_contact` >= `gcontact`.`last_failure`) OR
-                       (`gcontact`.`updated` >= `gcontact`.`last_failure`))
+                       NOT `gcontact`.`failed`
                        AND `gcontact`.`nurl` IN (select nurl from contact where uid = %d and self = 0 and blocked = 0 and hidden = 0 and id != %d) ",
                        intval($cid),
                        intval($uid),
@@ -337,7 +337,7 @@ class GContact
                        WHERE `glink`.`cid` = %d and `glink`.`uid` = %d
                                AND `contact`.`uid` = %d AND `contact`.`self` = 0 AND `contact`.`blocked` = 0
                                AND `contact`.`hidden` = 0 AND `contact`.`id` != %d
-                               AND ((`gcontact`.`last_contact` >= `gcontact`.`last_failure`) OR (`gcontact`.`updated` >= `gcontact`.`last_failure`))
+                               AND NOT `gcontact`.`failed`
                                $sql_extra LIMIT %d, %d",
                        intval($cid),
                        intval($uid),
@@ -396,7 +396,7 @@ class GContact
                        "SELECT count(*) as `total`
                        FROM `glink` INNER JOIN `gcontact` on `glink`.`gcid` = `gcontact`.`id`
                        where `glink`.`cid` = %d and `glink`.`uid` = %d AND
-                       ((`gcontact`.`last_contact` >= `gcontact`.`last_failure`) OR (`gcontact`.`updated` >= `gcontact`.`last_failure`))",
+                       NOT `gcontact`.`failed`",
                        intval($cid),
                        intval($uid)
                );
@@ -424,7 +424,7 @@ class GContact
                        INNER JOIN `gcontact` on `glink`.`gcid` = `gcontact`.`id`
                        LEFT JOIN `contact` ON `contact`.`nurl` = `gcontact`.`nurl` AND `contact`.`uid` = %d
                        WHERE `glink`.`cid` = %d AND `glink`.`uid` = %d AND
-                       ((`gcontact`.`last_contact` >= `gcontact`.`last_failure`) OR (`gcontact`.`updated` >= `gcontact`.`last_failure`))
+                       NOT `gcontact`.`failed`
                        ORDER BY `gcontact`.`name` ASC LIMIT %d, %d ",
                        intval($uid),
                        intval($cid),
@@ -471,7 +471,7 @@ class GContact
                        AND NOT `gcontact`.`name` IN (SELECT `name` FROM `contact` WHERE `uid` = %d)
                        AND NOT `gcontact`.`id` IN (SELECT `gcid` FROM `gcign` WHERE `uid` = %d)
                        AND `gcontact`.`updated` >= '%s' AND NOT `gcontact`.`hide`
-                       AND `gcontact`.`last_contact` >= `gcontact`.`last_failure`
+                       AND NOT `gcontact`.`failed`
                        AND `gcontact`.`network` IN (%s)
                        GROUP BY `glink`.`gcid` ORDER BY `gcontact`.`updated` DESC,`total` DESC LIMIT %d, %d",
                        intval($uid),
@@ -495,7 +495,7 @@ class GContact
                        AND NOT `gcontact`.`name` IN (SELECT `name` FROM `contact` WHERE `uid` = %d)
                        AND NOT `gcontact`.`id` IN (SELECT `gcid` FROM `gcign` WHERE `uid` = %d)
                        AND `gcontact`.`updated` >= '%s'
-                       AND `gcontact`.`last_contact` >= `gcontact`.`last_failure`
+                       AND NOT `gcontact`.`failed`
                        AND `gcontact`.`network` IN (%s)
                        ORDER BY rand() LIMIT %d, %d",
                        intval($uid),
@@ -642,7 +642,7 @@ class GContact
                $fields = ['name' => $contact['name'], 'nick' => $contact['nick'] ?? '', 'addr' => $contact['addr'] ?? '', 'network' => $contact['network'],
                        'url' => $contact['url'], 'nurl' => Strings::normaliseLink($contact['url']), 'photo' => $contact['photo'],
                        'created' => DateTimeFormat::utcNow(), 'updated' => DateTimeFormat::utcNow(), 'location' => $contact['location'],
-                       'about' => $contact['about'], 'hide' => $contact['hide'], 'generation' => $contact['generation']];
+                       'about' => $contact['about'], 'hide' => $contact['hide'], 'generation' => $contact['generation'], 'failed' => false];
 
                DBA::insert('gcontact', $fields);
 
@@ -681,7 +681,7 @@ class GContact
                }
 
                $public_contact = DBA::selectFirst('gcontact', [
-                       'name', 'nick', 'photo', 'location', 'about', 'addr', 'generation', 'birthday', 'keywords', 'gsid',
+                       'name', 'nick', 'photo', 'location', 'about', 'addr', 'generation', 'birthday', 'keywords', 'gsid', 'failed',
                        'contact-type', 'hide', 'nsfw', 'network', 'alias', 'notify', 'server_url', 'connect', 'updated', 'url'
                ], ['id' => $gcontact_id]);
 
@@ -787,7 +787,7 @@ class GContact
                                'location' => $contact['location'], 'about' => $contact['about'],
                                'generation' => $contact['generation'], 'updated' => $contact['updated'],
                                'server_url' => $contact['server_url'], 'connect' => $contact['connect'],
-                               'gsid' => $contact['gsid']
+                               'failed' => $contact['failed'], 'gsid' => $contact['gsid']
                        ];
 
                        DBA::update('gcontact', $updated, $condition, $fields);
@@ -851,13 +851,13 @@ class GContact
                        $noscrape = json_decode($curlResult->getBody(), true);
                        if (!empty($noscrape) && !empty($noscrape['updated'])) {
                                $noscrape['updated'] = DateTimeFormat::utc($noscrape['updated'], DateTimeFormat::MYSQL);
-                               $fields = ['last_contact' => DateTimeFormat::utcNow(), 'updated' => $noscrape['updated']];
+                               $fields = ['failed' => false, 'last_contact' => DateTimeFormat::utcNow(), 'updated' => $noscrape['updated']];
                                DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($data['url'])]);
                                return true;
                        }
                } elseif ($curlResult->isTimeout()) {
                        // On a timeout return the existing value, but mark the contact as failure
-                       $fields = ['last_failure' => DateTimeFormat::utcNow()];
+                       $fields = ['failed' => true, 'last_failure' => DateTimeFormat::utcNow()];
                        DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($data['url'])]);
                        return true;
                }
@@ -915,7 +915,7 @@ class GContact
                        return;
                }
 
-               $fields = ['last_contact' => DateTimeFormat::utcNow(), 'updated' => $last_updated];
+               $fields = ['failed' => false, 'last_contact' => DateTimeFormat::utcNow(), 'updated' => $last_updated];
                DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($data['url'])]);
        }
 
@@ -929,7 +929,7 @@ class GContact
                // Search for the newest entry in the feed
                $curlResult = Network::curl($data['poll']);
                if (!$curlResult->isSuccess()) {
-                       $fields = ['last_failure' => DateTimeFormat::utcNow()];
+                       $fields = ['failed' => true, 'last_failure' => DateTimeFormat::utcNow()];
                        DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($data['url'])]);
 
                        Logger::info("Profile wasn't reachable (no feed)", ['url' => $data['url']]);
@@ -970,7 +970,7 @@ class GContact
                        return;
                }
 
-               $fields = ['last_contact' => DateTimeFormat::utcNow(), 'updated' => $last_updated];
+               $fields = ['failed' => false, 'last_contact' => DateTimeFormat::utcNow(), 'updated' => $last_updated];
                DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($data['url'])]);
        }
        /**
@@ -1012,7 +1012,7 @@ class GContact
                $fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords',
                        'bd', 'contact-type', 'network', 'addr', 'notify', 'alias', 'archive', 'term-date',
                        'created', 'updated', 'avatar', 'success_update', 'failure_update', 'forum', 'prv',
-                       'baseurl', 'gsid', 'sensitive', 'unsearchable'];
+                       'baseurl', 'gsid', 'sensitive', 'unsearchable', 'failed'];
 
                $contact = DBA::selectFirst('contact', $fields, array_merge($condition, ['uid' => 0, 'network' => Protocol::FEDERATED]));
                if (!DBA::isResult($contact)) {
@@ -1022,7 +1022,7 @@ class GContact
                $fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords', 'generation',
                        'birthday', 'contact-type', 'network', 'addr', 'notify', 'alias', 'archived', 'archive_date',
                        'created', 'updated', 'photo', 'last_contact', 'last_failure', 'community', 'connect',
-                       'server_url', 'gsid', 'nsfw', 'hide', 'id'];
+                       'server_url', 'gsid', 'nsfw', 'hide', 'id', 'failed'];
 
                $old_gcontact = DBA::selectFirst('gcontact', $fields, ['nurl' => $contact['nurl']]);
                $do_insert = !DBA::isResult($old_gcontact);
@@ -1034,7 +1034,7 @@ class GContact
 
                // These fields are identical in both contact and gcontact
                $fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords', 'gsid',
-                       'contact-type', 'network', 'addr', 'notify', 'alias', 'created', 'updated'];
+                       'contact-type', 'network', 'addr', 'notify', 'alias', 'created', 'updated', 'failed'];
 
                foreach ($fields as $field) {
                        $gcontact[$field] = $contact[$field];
@@ -1100,13 +1100,14 @@ class GContact
                $data = Probe::uri($url, $force);
 
                if (in_array($data['network'], [Protocol::PHANTOM])) {
-                       $fields = ['last_failure' => DateTimeFormat::utcNow()];
+                       $fields = ['failed' => true, 'last_failure' => DateTimeFormat::utcNow()];
                        DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($url)]);
                        Logger::info('Invalid network for contact', ['url' => $data['url'], 'callstack' => System::callstack()]);
                        return false;
                }
 
                $data['server_url'] = $data['baseurl'];
+               $data['failed'] = false;
 
                self::update($data);
 
@@ -1268,7 +1269,7 @@ class GContact
 
                $r = DBA::select('gserver', ['nurl', 'url'], [
                        '`network` = ?
-                       AND `last_contact` >= `last_failure`
+                       AND NOT `failed`
                        AND `last_poco_query` < ?',
                        Protocol::OSTATUS,
                        $last_update
@@ -1419,8 +1420,8 @@ class GContact
        public static function getRandomUrl()
        {
                $r = DBA::selectFirst('gcontact', ['url'], [
-                       '`network` = ? 
-                       AND `last_contact` >= `last_failure`  
+                       '`network` = ?
+                       AND NOT `failed`
                        AND `updated` > ?',
                        Protocol::DFRN,
                        DateTimeFormat::utc('now - 1 month'),

diff --git a/src/Model/GServer.php b/src/Model/GServer.php
index fdbc7750985a3a3e2bb49635458727dcaa3de50f..3d268c37b91f699bb0c4950f2c64636b9d828901 100644 (file)
--- a/src/Model/GServer.php
+++ b/src/Model/GServer.php
@@ -235,14 +235,14 @@ class GServer
        private static function setFailure(string $url)
        {
                if (DBA::exists('gserver', ['nurl' => Strings::normaliseLink($url)])) {
-                       DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow(), 'detection-method' => null],
+                       DBA::update('gserver', ['failed' => true, 'last_failure' => DateTimeFormat::utcNow(), 'detection-method' => null],
                        ['nurl' => Strings::normaliseLink($url)]);
                        Logger::info('Set failed status for existing server', ['url' => $url]);
                        return;
                }
                DBA::insert('gserver', ['url' => $url, 'nurl' => Strings::normaliseLink($url),
                        'network' => Protocol::PHANTOM, 'created' => DateTimeFormat::utcNow(),
-                       'last_failure' => DateTimeFormat::utcNow()]);
+                       'failed' => true, 'last_failure' => DateTimeFormat::utcNow()]);
                Logger::info('Set failed status for new server', ['url' => $url]);
        }
 
@@ -303,7 +303,8 @@ class GServer
 
                // If the URL missmatches, then we mark the old entry as failure
                if ($url != $original_url) {
-                       DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($original_url)]);
+                       DBA::update('gserver', ['failed' => true, 'last_failure' => DateTimeFormat::utcNow()],
+                               ['nurl' => Strings::normaliseLink($original_url)]);
                }
 
                // When a nodeinfo is present, we don't need to dig further
@@ -449,6 +450,7 @@ class GServer
                }
 
                $serverdata['last_contact'] = DateTimeFormat::utcNow();
+               $serverdata['failed'] = false;
 
                $gserver = DBA::selectFirst('gserver', ['network'], ['nurl' => Strings::normaliseLink($url)]);
                if (!DBA::isResult($gserver)) {
@@ -1585,7 +1587,7 @@ class GServer
 
                $gservers = DBA::p("SELECT `id`, `url`, `nurl`, `network`, `poco`
                        FROM `gserver`
-                       WHERE `last_contact` >= `last_failure`
+                       WHERE NOT `failed`
                        AND `poco` != ''
                        AND `last_poco_query` < ?
                        ORDER BY RAND()", $last_update

diff --git a/src/Model/Item.php b/src/Model/Item.php
index 13a4a68380667554855d2a8d00f32fcb514ad200..e31097f53cc48f2d75b02a80e0cedc8f2a60f0fd 100644 (file)
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -2415,7 +2415,7 @@ class Item
                }
 
                /// @todo On private posts we could obfuscate the date
-               $update = ($arr['private'] != self::PRIVATE);
+               $update = ($arr['private'] != self::PRIVATE) || in_array($arr['network'], Protocol::FEDERATED);
 
                // Is it a forum? Then we don't care about the rules from above
                if (!$update && in_array($arr["network"], [Protocol::ACTIVITYPUB, Protocol::DFRN]) && ($arr["parent-uri"] === $arr["uri"])) {
@@ -2433,15 +2433,15 @@ class Item
                        } else { 
                                $condition = ['id' => $arr['contact-id'], 'self' => false];
                        }
-                       DBA::update('contact', ['success_update' => $arr['received'], 'last-item' => $arr['received']], $condition);
+                       DBA::update('contact', ['failed' => false, 'success_update' => $arr['received'], 'last-item' => $arr['received']], $condition);
                }
                // Now do the same for the system wide contacts with uid=0
                if ($arr['private'] != self::PRIVATE) {
-                       DBA::update('contact', ['success_update' => $arr['received'], 'last-item' => $arr['received']],
+                       DBA::update('contact', ['failed' => false, 'success_update' => $arr['received'], 'last-item' => $arr['received']],
                                ['id' => $arr['owner-id']]);
 
                        if ($arr['owner-id'] != $arr['author-id']) {
-                               DBA::update('contact', ['success_update' => $arr['received'], 'last-item' => $arr['received']],
+                               DBA::update('contact', ['failed' => false, 'success_update' => $arr['received'], 'last-item' => $arr['received']],
                                        ['id' => $arr['author-id']]);
                        }
                }

diff --git a/src/Module/Admin/Federation.php b/src/Module/Admin/Federation.php
index 928a286b14ebf7a42e9ad27e632e256e37f42271..fd60b071526b88fc65e3b8106342db0554c72548 100644 (file)
--- a/src/Module/Admin/Federation.php
+++ b/src/Module/Admin/Federation.php
@@ -64,14 +64,14 @@ class Federation extends BaseAdmin
 
                $gservers = DBA::p("SELECT COUNT(*) AS `total`, SUM(`registered-users`) AS `users`, `platform`,
                        ANY_VALUE(`network`) AS `network`, MAX(`version`) AS `version`
-                       FROM `gserver` WHERE `last_contact` >= `last_failure` GROUP BY `platform`");
+                       FROM `gserver` WHERE NOT `failed` GROUP BY `platform`");
                while ($gserver = DBA::fetch($gservers)) {
                        $total += $gserver['total'];
                        $users += $gserver['users'];
 
                        $versionCounts = [];
                        $versions = DBA::p("SELECT COUNT(*) AS `total`, `version` FROM `gserver`
-                               WHERE `last_contact` >= `last_failure` AND `platform` = ?
+                               WHERE NOT `failed` AND `platform` = ?
                                GROUP BY `version` ORDER BY `version`", $gserver['platform']);
                        while ($version = DBA::fetch($versions)) {
                                $version['version'] = str_replace(["\n", "\r", "\t"], " ", $version['version']);

diff --git a/src/Module/Api/Mastodon/Instance/Peers.php b/src/Module/Api/Mastodon/Instance/Peers.php
index 82f08cbad0eccad9c5f6bc1a7b4caaa5a3e58a1d..537d25e28c3c93ccc1c7d2abb18eb82b4ed5b4af 100644 (file)
--- a/src/Module/Api/Mastodon/Instance/Peers.php
+++ b/src/Module/Api/Mastodon/Instance/Peers.php
@@ -42,7 +42,7 @@ class Peers extends BaseApi
                $return = [];
 
                // We only select for Friendica and ActivityPub servers, since it is expected to only deliver AP compatible systems here.
-               $instances = DBA::select('gserver', ['url'], ["`network` in (?, ?) AND `last_contact` >= `last_failure`", Protocol::DFRN, Protocol::ACTIVITYPUB]);
+               $instances = DBA::select('gserver', ['url'], ["`network` in (?, ?) AND NOT `failed`", Protocol::DFRN, Protocol::ACTIVITYPUB]);
                while ($instance = DBA::fetch($instances)) {
                        $urldata = parse_url($instance['url']);
                        unset($urldata['scheme']);

diff --git a/src/Object/Api/Mastodon/Stats.php b/src/Object/Api/Mastodon/Stats.php
index 8677cf04252b9bc0cf81964142c3103c9cfd78f0..6ead5267294ced3aec93e555c306431eadd8dd5a 100644 (file)
--- a/src/Object/Api/Mastodon/Stats.php
+++ b/src/Object/Api/Mastodon/Stats.php
@@ -51,7 +51,7 @@ class Stats extends BaseEntity
                if (!empty(DI::config()->get('system', 'nodeinfo'))) {
                        $stats->user_count = intval(DI::config()->get('nodeinfo', 'total_users'));
                        $stats->status_count = DI::config()->get('nodeinfo', 'local_posts') + DI::config()->get('nodeinfo', 'local_comments');
-                       $stats->domain_count = DBA::count('gserver', ["`network` in (?, ?) AND `last_contact` >= `last_failure`", Protocol::DFRN, Protocol::ACTIVITYPUB]);
+                       $stats->domain_count = DBA::count('gserver', ["`network` in (?, ?) AND NOT `failed`", Protocol::DFRN, Protocol::ACTIVITYPUB]);
                }
                return $stats;
        }

diff --git a/src/Protocol/PortableContact.php b/src/Protocol/PortableContact.php
index f255347c1284e62e87ad963771dabec48c55c6fa..d66676cac5699eb0fac6e802bd5717a5685a3535 100644 (file)
--- a/src/Protocol/PortableContact.php
+++ b/src/Protocol/PortableContact.php
@@ -228,7 +228,7 @@ class PortableContact
        {
                $r = q(
                        "SELECT `url`, `site_name` AS `displayName`, `network`, `platform`, `version` FROM `gserver`
-                       WHERE `network` IN ('%s', '%s', '%s') AND `last_contact` > `last_failure`
+                       WHERE `network` IN ('%s', '%s', '%s') AND NOT `failed`
                        ORDER BY `last_contact`
                        LIMIT 1000",
                        DBA::escape(Protocol::DFRN),

diff --git a/src/Worker/OnePoll.php b/src/Worker/OnePoll.php
index a2659f4e76b0237c68fc83824cc0833d5b6f551a..fa8f748334e431a7495a25ebf44c76e633fbf8dd 100644 (file)
--- a/src/Worker/OnePoll.php
+++ b/src/Worker/OnePoll.php
@@ -97,7 +97,7 @@ class OnePoll
                }
 
                // load current friends if possible.
-               if (!empty($contact['poco']) && ($contact['success_update'] > $contact['failure_update'])) {
+               if (!empty($contact['poco']) && !$contact['failed']) {
                        if (!DBA::exists('glink', ["`cid` = ? AND updated > UTC_TIMESTAMP() - INTERVAL 1 DAY", $contact['id']])) {
                                PortableContact::loadWorker($contact['id'], $importer_uid, 0, $contact['poco']);
                        }
@@ -165,7 +165,7 @@ class OnePoll
                        if (!strstr($xml, '<')) {
                                Logger::log('post_handshake: response from ' . $url . ' did not contain XML.');
 
-                               $fields = ['last-update' => $updated, 'failure_update' => $updated];
+                               $fields = ['failed' => true, 'last-update' => $updated, 'failure_update' => $updated];
                                self::updateContact($contact, $fields);
                                Contact::markForArchival($contact);
                                return;
@@ -213,10 +213,10 @@ class OnePoll
                                }
                        }
 
-                       self::updateContact($contact, ['last-update' => $updated, 'success_update' => $updated]);
+                       self::updateContact($contact, ['failed' => false, 'last-update' => $updated, 'success_update' => $updated]);
                        Contact::unmarkForArchival($contact);
                } elseif (in_array($contact["network"], [Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, Protocol::FEED])) {
-                       self::updateContact($contact, ['last-update' => $updated, 'failure_update' => $updated]);
+                       self::updateContact($contact, ['failed' => true, 'last-update' => $updated, 'failure_update' => $updated]);
                        Contact::markForArchival($contact);
                } else {
                        self::updateContact($contact, ['last-update' => $updated]);
@@ -244,8 +244,14 @@ class OnePoll
         */
        private static function updateContact(array $contact, array $fields)
        {
+               // Update the user's contact
                DBA::update('contact', $fields, ['id' => $contact['id']]);
+
+               // Update the public contact
                DBA::update('contact', $fields, ['uid' => 0, 'nurl' => $contact['nurl']]);
+
+               // Update the rest of the contacts that aren't polled
+               DBA::update('contact', $fields, ['rel' => Contact::FOLLOWER, 'nurl' => $contact['nurl']]);
        }
 
        /**
@@ -289,7 +295,7 @@ class OnePoll
 
                if (!$curlResult->isSuccess() && ($curlResult->getErrorNumber() == CURLE_OPERATION_TIMEDOUT)) {
                        // set the last-update so we don't keep polling
-                       self::updateContact($contact, ['last-update' => $updated]);
+                       self::updateContact($contact, ['failed' => true, 'last-update' => $updated]);
                        Contact::markForArchival($contact);
                        Logger::log('Contact archived');
                        return false;
@@ -307,7 +313,7 @@ class OnePoll
                        Logger::log("$url appears to be dead - marking for death ");
 
                        // set the last-update so we don't keep polling
-                       $fields = ['last-update' => $updated, 'failure_update' => $updated];
+                       $fields = ['failed' => true, 'last-update' => $updated, 'failure_update' => $updated];
                        self::updateContact($contact, $fields);
                        Contact::markForArchival($contact);
                        return false;
@@ -316,7 +322,7 @@ class OnePoll
                if (!strstr($handshake_xml, '<')) {
                        Logger::log('response from ' . $url . ' did not contain XML.');
 
-                       $fields = ['last-update' => $updated, 'failure_update' => $updated];
+                       $fields = ['failed' => true, 'last-update' => $updated, 'failure_update' => $updated];
                        self::updateContact($contact, $fields);
                        Contact::markForArchival($contact);
                        return false;
@@ -327,7 +333,7 @@ class OnePoll
                if (!is_object($res)) {
                        Logger::info('Unparseable response', ['url' => $url]);
 
-                       $fields = ['last-update' => $updated, 'failure_update' => $updated];
+                       $fields = ['failed' => true, 'last-update' => $updated, 'failure_update' => $updated];
                        self::updateContact($contact, $fields);
                        Contact::markForArchival($contact);
                        return false;
@@ -338,7 +344,7 @@ class OnePoll
                        Logger::log("$url replied status 1 - marking for death ");
 
                        // set the last-update so we don't keep polling
-                       $fields = ['last-update' => $updated, 'failure_update' => $updated];
+                       $fields = ['failed' => true, 'last-update' => $updated, 'failure_update' => $updated];
                        self::updateContact($contact, $fields);
                        Contact::markForArchival($contact);
                } elseif ($contact['term-date'] > DBA::NULL_DATETIME) {
@@ -417,7 +423,7 @@ class OnePoll
                // Will only do this once per notify-enabled OStatus contact
                // or if relationship changes
 
-               $stat_writeable = ((($contact['notify']) && ($contact['rel'] == Contact::FOLLOWER || $contact['rel'] == Contact::FRIEND)) ? 1 : 0);
+               $stat_writeable = $contact['notify'] && ($contact['rel'] == Contact::FOLLOWER || $contact['rel'] == Contact::FRIEND);
 
                // Contacts from OStatus are always writable
                if ($protocol === Protocol::OSTATUS) {
@@ -443,7 +449,7 @@ class OnePoll
 
                if ($curlResult->isTimeout()) {
                        // set the last-update so we don't keep polling
-                       self::updateContact($contact, ['last-update' => $updated]);
+                       self::updateContact($contact, ['failed' => true, 'last-update' => $updated]);
                        Contact::markForArchival($contact);
                        Logger::log('Contact archived');
                        return false;
@@ -467,7 +473,7 @@ class OnePoll
                $mail_disabled = ((function_exists('imap_open') && !DI::config()->get('system', 'imap_disabled')) ? 0 : 1);
                if ($mail_disabled) {
                        // set the last-update so we don't keep polling
-                       self::updateContact($contact, ['last-update' => $updated]);
+                       self::updateContact($contact, ['failed' => true, 'last-update' => $updated]);
                        Contact::markForArchival($contact);
                        Logger::log('Contact archived');
                        return;

diff --git a/src/Worker/SearchDirectory.php b/src/Worker/SearchDirectory.php
index 2ffe6120ec56e950b24278ca776079ae3b2e1dc7..afe54e5fb1ee4fb70ec3ad6808447e07fbfeb69f 100644 (file)
--- a/src/Worker/SearchDirectory.php
+++ b/src/Worker/SearchDirectory.php
@@ -58,12 +58,11 @@ class SearchDirectory
                if (!empty($j->results)) {
                        foreach ($j->results as $jj) {
                                // Check if the contact already exists
-                               $gcontact = DBA::selectFirst('gcontact', ['id', 'last_contact', 'last_failure', 'updated'], ['nurl' => Strings::normaliseLink($jj->url)]);
+                               $gcontact = DBA::selectFirst('gcontact', ['failed'], ['nurl' => Strings::normaliseLink($jj->url)]);
                                if (DBA::isResult($gcontact)) {
                                        Logger::info('Profile already exists', ['profile' => $jj->url, 'search' => $search]);
 
-                                       if (($gcontact['last_contact'] < $gcontact['last_failure']) &&
-                                               ($gcontact['updated'] < $gcontact['last_failure'])) {
+                                       if ($gcontact['failed']) {
                                                continue;
                                        }
 

diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php
index 344a21fc5d326b5b4a354ad84135fe5edd324604..14a52aa9ad5aa5fb0b0648b2cf06bcb36f729da1 100755 (executable)
--- a/static/dbstructure.config.php
+++ b/static/dbstructure.config.php
@@ -54,7 +54,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-       define('DB_UPDATE_VERSION', 1356);
+       define('DB_UPDATE_VERSION', 1357);
 }
 
 return [
@@ -82,6 +82,7 @@ return [
                        "last_poco_query" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""],
                        "last_contact" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""],
                        "last_failure" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""],
+                       "failed" => ["type" => "boolean", "comment" => "Connection failed"],
                ],
                "indexes" => [
                        "PRIMARY" => ["id"],
@@ -151,6 +152,7 @@ return [
                        "last-update" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last try to update the contact info"],
                        "success_update" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last successful contact update"],
                        "failure_update" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last failed update"],
+                       "failed" => ["type" => "boolean", "comment" => "Connection failed"],
                        "name-date" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""],
                        "uri-date" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""],
                        "avatar-date" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""],
@@ -560,6 +562,7 @@ return [
                        "last_contact" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""],
                        "last_failure" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""],
                        "last_discovery" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last contact discovery"],
+                       "failed" => ["type" => "boolean", "comment" => "Connection failed"],
                        "archive_date" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""],
                        "archived" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],
                        "location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],

diff --git a/update.php b/update.php
index 1a9a60ac65b1e6c1e3105c750062451b4b64a30a..a4d71bc4fde2b30f11e3e0a0fe71f7a53439fa44 100644 (file)
--- a/update.php
+++ b/update.php
@@ -535,3 +535,43 @@ function update_1354()
        }
        return Update::SUCCESS;
 }
+
+function update_1357()
+{
+       if (!DBA::e("UPDATE `contact` SET `failed` = true WHERE `success_update` < `failure_update` AND `failed` IS NULL")) {
+               return Update::FAILED;
+       }
+
+       if (!DBA::e("UPDATE `contact` SET `failed` = false WHERE `success_update` > `failure_update` AND `failed` IS NULL")) {
+               return Update::FAILED;
+       }
+
+       if (!DBA::e("UPDATE `contact` SET `failed` = false WHERE `updated` > `failure_update` AND `failed` IS NULL")) {
+               return Update::FAILED;
+       }
+
+       if (!DBA::e("UPDATE `contact` SET `failed` = false WHERE `last-item` > `failure_update` AND `failed` IS NULL")) {
+               return Update::FAILED;
+       }
+
+       if (!DBA::e("UPDATE `gserver` SET `failed` = true WHERE `last_contact` < `last_failure` AND `failed` IS NULL")) {
+               return Update::FAILED;
+       }
+
+       if (!DBA::e("UPDATE `gserver` SET `failed` = false WHERE `last_contact` > `last_failure` AND `failed` IS NULL")) {
+               return Update::FAILED;
+       }
+
+       if (!DBA::e("UPDATE `gcontact` SET `failed` = true WHERE `last_contact` < `last_failure` AND `failed` IS NULL")) {
+               return Update::FAILED;
+       }
+
+       if (!DBA::e("UPDATE `gcontact` SET `failed` = false WHERE `last_contact` > `last_failure` AND `failed` IS NULL")) {
+               return Update::FAILED;
+       }
+
+       if (!DBA::e("UPDATE `gcontact` SET `failed` = false WHERE `updated` > `last_failure` AND `failed` IS NULL")) {
+               return Update::FAILED;
+       }
+       return Update::SUCCESS;
+}