X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=src%2FWorker%2FOnePoll.php;h=5fc65b211a1d4f4414d896c64ef1d64c5f737f97;hb=3a1fef43b237a4fcb75d7880aa81808a66c56f2d;hp=9af4be9bed32185f964962adb43956151daf1ea3;hpb=7b0a9ffd8923a26f416ce67a285b692c8f84ad3f;p=friendica.git diff --git a/src/Worker/OnePoll.php b/src/Worker/OnePoll.php index 9af4be9bed..5fc65b211a 100644 --- a/src/Worker/OnePoll.php +++ b/src/Worker/OnePoll.php @@ -1,24 +1,38 @@ . + * */ + namespace Friendica\Worker; -use Friendica\BaseObject; -use Friendica\Content\Text\BBCode; -use Friendica\Core\Config; use Friendica\Core\Logger; -use Friendica\Core\PConfig; use Friendica\Core\Protocol; use Friendica\Database\DBA; +use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\Item; use Friendica\Model\User; +use Friendica\Protocol\Activity; use Friendica\Protocol\ActivityPub; use Friendica\Protocol\Email; -use Friendica\Protocol\PortableContact; +use Friendica\Protocol\Feed; use Friendica\Util\DateTimeFormat; -use Friendica\Util\Network; use Friendica\Util\Strings; use Friendica\Util\XML; @@ -39,9 +53,6 @@ class OnePoll return; } - if ($force) { - Contact::updateFromProbe($contact_id, true); - } $contact = DBA::selectFirst('contact', [], ['id' => $contact_id]); if (!DBA::isResult($contact)) { @@ -49,6 +60,16 @@ class OnePoll return; } + if (($contact['network'] != Protocol::MAIL) && $force) { + Contact::updateFromProbe($contact_id); + } + + // Special treatment for wrongly detected local contacts + if (!$force && ($contact['network'] != Protocol::DFRN) && Contact::isLocalById($contact_id)) { + Contact::updateFromProbe($contact_id, Protocol::DFRN); + $contact = DBA::selectFirst('contact', [], ['id' => $contact_id]); + } + if (($contact['network'] == Protocol::DFRN) && !Contact::isLegacyDFRNContact($contact)) { $protocol = Protocol::ACTIVITYPUB; } else { @@ -73,15 +94,8 @@ class OnePoll $contact = DBA::selectFirst('contact', [], ['id' => $contact_id]); } - // load current friends if possible. - if (!empty($contact['poco']) && ($contact['success_update'] > $contact['failure_update'])) { - if (!DBA::exists('glink', ["`cid` = ? AND updated > UTC_TIMESTAMP() - INTERVAL 1 DAY", $contact['id']])) { - PortableContact::loadWorker($contact['id'], $importer_uid, 0, $contact['poco']); - } - } - // Don't poll if polling is deactivated (But we poll feeds and mails anyway) - if (!in_array($protocol, [Protocol::FEED, Protocol::MAIL]) && Config::get('system', 'disable_polling')) { + if (!in_array($protocol, [Protocol::FEED, Protocol::MAIL]) && DI::config()->get('system', 'disable_polling')) { Logger::log('Polling is disabled'); // set the last-update so we don't keep polling @@ -89,6 +103,15 @@ class OnePoll return; } + // Don't poll local contacts + if (User::getIdForURL($contact['url'])) { + Logger::info('Local contacts are not polled', ['id' => $contact['id']]); + + // set the last-update so we don't keep polling + DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]); + return; + } + // We don't poll AP contacts by now if ($protocol === Protocol::ACTIVITYPUB) { Logger::log("Don't poll AP contact"); @@ -112,7 +135,7 @@ class OnePoll $xml = false; if ($contact['subhub']) { - $poll_interval = Config::get('system', 'pushpoll_frequency', 3); + $poll_interval = DI::config()->get('system', 'pushpoll_frequency', 3); $contact['priority'] = intval($poll_interval); $hub_update = false; @@ -142,7 +165,7 @@ class OnePoll if (!strstr($xml, '<')) { Logger::log('post_handshake: response from ' . $url . ' did not contain XML.'); - $fields = ['last-update' => $updated, 'failure_update' => $updated]; + $fields = ['failed' => true, 'last-update' => $updated, 'failure_update' => $updated]; self::updateContact($contact, $fields); Contact::markForArchival($contact); return; @@ -151,11 +174,11 @@ class OnePoll Logger::log("Consume feed of contact ".$contact['id']); - consume_feed($xml, $importer, $contact, $hub); + Feed::consume($xml, $importer, $contact, $hub); // do it a second time for DFRN so that any children find their parents. if ($protocol === Protocol::DFRN) { - consume_feed($xml, $importer, $contact, $hub); + Feed::consume($xml, $importer, $contact, $hub); } $hubmode = 'subscribe'; @@ -185,15 +208,15 @@ class OnePoll continue; } - subscribe_to_hub($h, $importer, $contact, $hubmode); + self::subscribeToHub($h, $importer, $contact, $hubmode); } } } - self::updateContact($contact, ['last-update' => $updated, 'success_update' => $updated]); + self::updateContact($contact, ['failed' => false, 'last-update' => $updated, 'success_update' => $updated]); Contact::unmarkForArchival($contact); } elseif (in_array($contact["network"], [Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, Protocol::FEED])) { - self::updateContact($contact, ['last-update' => $updated, 'failure_update' => $updated]); + self::updateContact($contact, ['failed' => true, 'last-update' => $updated, 'failure_update' => $updated]); Contact::markForArchival($contact); } else { self::updateContact($contact, ['last-update' => $updated]); @@ -213,7 +236,7 @@ class OnePoll } /** - * @brief Updates a personal contact entry and the public contact entry + * Updates a personal contact entry and the public contact entry * * @param array $contact The personal contact entry * @param array $fields The fields that are updated @@ -221,12 +244,18 @@ class OnePoll */ private static function updateContact(array $contact, array $fields) { + // Update the user's contact DBA::update('contact', $fields, ['id' => $contact['id']]); + + // Update the public contact DBA::update('contact', $fields, ['uid' => 0, 'nurl' => $contact['nurl']]); + + // Update the rest of the contacts that aren't polled + DBA::update('contact', $fields, ['rel' => Contact::FOLLOWER, 'nurl' => $contact['nurl']]); } /** - * @brief Poll DFRN contacts + * Poll DFRN contacts * * @param array $contact The personal contact entry * @param string $updated The updated date @@ -262,11 +291,11 @@ class OnePoll . '&type=data&last_update=' . $last_update . '&perm=' . $perm; - $curlResult = Network::curl($url); + $curlResult = DI::httpRequest()->get($url); if (!$curlResult->isSuccess() && ($curlResult->getErrorNumber() == CURLE_OPERATION_TIMEDOUT)) { // set the last-update so we don't keep polling - self::updateContact($contact, ['last-update' => $updated]); + self::updateContact($contact, ['failed' => true, 'last-update' => $updated]); Contact::markForArchival($contact); Logger::log('Contact archived'); return false; @@ -284,7 +313,7 @@ class OnePoll Logger::log("$url appears to be dead - marking for death "); // set the last-update so we don't keep polling - $fields = ['last-update' => $updated, 'failure_update' => $updated]; + $fields = ['failed' => true, 'last-update' => $updated, 'failure_update' => $updated]; self::updateContact($contact, $fields); Contact::markForArchival($contact); return false; @@ -293,7 +322,7 @@ class OnePoll if (!strstr($handshake_xml, '<')) { Logger::log('response from ' . $url . ' did not contain XML.'); - $fields = ['last-update' => $updated, 'failure_update' => $updated]; + $fields = ['failed' => true, 'last-update' => $updated, 'failure_update' => $updated]; self::updateContact($contact, $fields); Contact::markForArchival($contact); return false; @@ -301,12 +330,21 @@ class OnePoll $res = XML::parseString($handshake_xml); + if (!is_object($res)) { + Logger::info('Unparseable response', ['url' => $url]); + + $fields = ['failed' => true, 'last-update' => $updated, 'failure_update' => $updated]; + self::updateContact($contact, $fields); + Contact::markForArchival($contact); + return false; + } + if (intval($res->status) == 1) { // we may not be friends anymore. Will keep trying for one month. Logger::log("$url replied status 1 - marking for death "); // set the last-update so we don't keep polling - $fields = ['last-update' => $updated, 'failure_update' => $updated]; + $fields = ['failed' => true, 'last-update' => $updated, 'failure_update' => $updated]; self::updateContact($contact, $fields); Contact::markForArchival($contact); } elseif ($contact['term-date'] > DBA::NULL_DATETIME) { @@ -367,11 +405,11 @@ class OnePoll $postvars['dfrn_version'] = DFRN_PROTOCOL_VERSION; $postvars['perm'] = 'rw'; - return Network::post($contact['poll'], $postvars)->getBody(); + return DI::httpRequest()->post($contact['poll'], $postvars)->getBody(); } /** - * @brief Poll Feed/OStatus contacts + * Poll Feed/OStatus contacts * * @param array $contact The personal contact entry * @param string $protocol The used protocol of the contact @@ -385,7 +423,7 @@ class OnePoll // Will only do this once per notify-enabled OStatus contact // or if relationship changes - $stat_writeable = ((($contact['notify']) && ($contact['rel'] == Contact::FOLLOWER || $contact['rel'] == Contact::FRIEND)) ? 1 : 0); + $stat_writeable = $contact['notify'] && ($contact['rel'] == Contact::FOLLOWER || $contact['rel'] == Contact::FRIEND); // Contacts from OStatus are always writable if ($protocol === Protocol::OSTATUS) { @@ -406,12 +444,12 @@ class OnePoll } $cookiejar = tempnam(get_temppath(), 'cookiejar-onepoll-'); - $curlResult = Network::curl($contact['poll'], false, ['cookiejar' => $cookiejar]); + $curlResult = DI::httpRequest()->get($contact['poll'], false, ['cookiejar' => $cookiejar]); unlink($cookiejar); if ($curlResult->isTimeout()) { // set the last-update so we don't keep polling - self::updateContact($contact, ['last-update' => $updated]); + self::updateContact($contact, ['failed' => true, 'last-update' => $updated]); Contact::markForArchival($contact); Logger::log('Contact archived'); return false; @@ -421,7 +459,7 @@ class OnePoll } /** - * @brief Poll Mail contacts + * Poll Mail contacts * * @param array $contact The personal contact entry * @param integer $importer_uid The UID of the importer @@ -432,10 +470,10 @@ class OnePoll { Logger::log("Mail: Fetching for ".$contact['addr'], Logger::DEBUG); - $mail_disabled = ((function_exists('imap_open') && !Config::get('system', 'imap_disabled')) ? 0 : 1); + $mail_disabled = ((function_exists('imap_open') && !DI::config()->get('system', 'imap_disabled')) ? 0 : 1); if ($mail_disabled) { // set the last-update so we don't keep polling - self::updateContact($contact, ['last-update' => $updated]); + self::updateContact($contact, ['failed' => true, 'last-update' => $updated]); Contact::markForArchival($contact); Logger::log('Contact archived'); return; @@ -484,8 +522,10 @@ class OnePoll Logger::log("Mail: Parsing mail ".$msg_uid, Logger::DATA); $datarray = []; - $datarray['verb'] = ACTIVITY_POST; - $datarray['object-type'] = ACTIVITY_OBJ_NOTE; + $datarray['uid'] = $importer_uid; + $datarray['contact-id'] = $contact['id']; + $datarray['verb'] = Activity::POST; + $datarray['object-type'] = Activity\ObjectType::NOTE; $datarray['network'] = Protocol::MAIL; // $meta = Email::messageMeta($mbox, $msg_uid); @@ -529,9 +569,9 @@ class OnePoll } // look for a 'references' or an 'in-reply-to' header and try to match with a parent item we have locally. - $raw_refs = ((property_exists($meta, 'references')) ? str_replace("\t", '', $meta->references) : ''); + $raw_refs = (property_exists($meta, 'references') ? str_replace("\t", '', $meta->references) : ''); if (!trim($raw_refs)) { - $raw_refs = ((property_exists($meta, 'in_reply_to')) ? str_replace("\t", '', $meta->in_reply_to) : ''); + $raw_refs = (property_exists($meta, 'in_reply_to') ? str_replace("\t", '', $meta->in_reply_to) : ''); } $raw_refs = trim($raw_refs); // Don't allow a blank reference in $refs_arr @@ -550,7 +590,7 @@ class OnePoll } // Decoding the header - $subject = imap_mime_header_decode($meta->subject); + $subject = imap_mime_header_decode($meta->subject ?? ''); $datarray['title'] = ""; foreach ($subject as $subpart) { if ($subpart->charset != "default") { @@ -586,36 +626,33 @@ class OnePoll $datarray['parent-uri'] = $datarray['uri']; } - $r = Email::getMessage($mbox, $msg_uid, $reply); - if (!$r) { - Logger::log("Mail: can't fetch msg ".$msg_uid." for ".$mailconf['user']); - continue; - } - $datarray['body'] = Strings::escapeHtml($r['body']); - $datarray['body'] = BBCode::limitBodySize($datarray['body']); + $headers = imap_headerinfo($mbox, $meta->msgno); - Logger::log("Mail: Importing ".$msg_uid." for ".$mailconf['user']); + $object = []; - /// @TODO Adding a gravatar for the original author would be cool + if (!empty($headers->from)) { + $object['from'] = $headers->from; + } - $from = imap_mime_header_decode($meta->from); - $fromdecoded = ""; - foreach ($from as $frompart) { - if ($frompart->charset != "default") { - $fromdecoded .= iconv($frompart->charset, 'UTF-8//IGNORE', $frompart->text); - } else { - $fromdecoded .= $frompart->text; - } + if (!empty($headers->to)) { + $object['to'] = $headers->to; + } + + if (!empty($headers->reply_to)) { + $object['reply_to'] = $headers->reply_to; } - $fromarr = imap_rfc822_parse_adrlist($fromdecoded, BaseObject::getApp()->getHostName()); + if (!empty($headers->sender)) { + $object['sender'] = $headers->sender; + } - $frommail = $fromarr[0]->mailbox."@".$fromarr[0]->host; + if (!empty($object)) { + $datarray['object'] = json_encode($object); + } - if (isset($fromarr[0]->personal)) { - $fromname = $fromarr[0]->personal; - } else { - $fromname = $frommail; + $fromname = $frommail = $headers->from[0]->mailbox . '@' . $headers->from[0]->host; + if (!empty($headers->from[0]->personal)) { + $fromname = $headers->from[0]->personal; } $datarray['author-name'] = $fromname; @@ -626,16 +663,23 @@ class OnePoll $datarray['owner-link'] = "mailto:".$contact['addr']; $datarray['owner-avatar'] = $contact['photo']; - $datarray['uid'] = $importer_uid; - $datarray['contact-id'] = $contact['id']; if ($datarray['parent-uri'] === $datarray['uri']) { - $datarray['private'] = 1; + $datarray['private'] = Item::PRIVATE; } - if (!PConfig::get($importer_uid, 'system', 'allow_public_email_replies')) { - $datarray['private'] = 1; + + if (!DI::pConfig()->get($importer_uid, 'system', 'allow_public_email_replies')) { + $datarray['private'] = Item::PRIVATE; $datarray['allow_cid'] = '<' . $contact['id'] . '>'; } + $datarray = Email::getMessage($mbox, $msg_uid, $reply, $datarray); + if (empty($datarray['body'])) { + Logger::log("Mail: can't fetch msg ".$msg_uid." for ".$mailconf['user']); + continue; + } + + Logger::log("Mail: Importing ".$msg_uid." for ".$mailconf['user']); + Item::insert($datarray); switch ($mailconf['action']) { @@ -664,7 +708,62 @@ class OnePoll Logger::log("Mail: no mails for ".$mailconf['user']); } + self::updateContact($contact, ['failed' => false, 'last-update' => $updated, 'success_update' => $updated]); + Contact::unmarkForArchival($contact); + Logger::log("Mail: closing connection for ".$mailconf['user']); imap_close($mbox); } + + + /** + * @param string $url + * @param array $importer + * @param array $contact + * @param string $hubmode + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + */ + private static function subscribeToHub(string $url, array $importer, array $contact, $hubmode = 'subscribe') + { + /* + * Diaspora has different message-ids in feeds than they do + * through the direct Diaspora protocol. If we try and use + * the feed, we'll get duplicates. So don't. + */ + if ($contact['network'] === Protocol::DIASPORA) { + return; + } + + // Without an importer we don't have a user id - so we quit + if (empty($importer)) { + return; + } + + $user = DBA::selectFirst('user', ['nickname'], ['uid' => $importer['uid']]); + + // No user, no nickname, we quit + if (!DBA::isResult($user)) { + return; + } + + $push_url = DI::baseUrl() . '/pubsub/' . $user['nickname'] . '/' . $contact['id']; + + // Use a single verify token, even if multiple hubs + $verify_token = ((strlen($contact['hub-verify'])) ? $contact['hub-verify'] : Strings::getRandomHex()); + + $params = 'hub.mode=' . $hubmode . '&hub.callback=' . urlencode($push_url) . '&hub.topic=' . urlencode($contact['poll']) . '&hub.verify=async&hub.verify_token=' . $verify_token; + + Logger::log('subscribe_to_hub: ' . $hubmode . ' ' . $contact['name'] . ' to hub ' . $url . ' endpoint: ' . $push_url . ' with verifier ' . $verify_token); + + if (!strlen($contact['hub-verify']) || ($contact['hub-verify'] != $verify_token)) { + DBA::update('contact', ['hub-verify' => $verify_token], ['id' => $contact['id']]); + } + + $postResult = DI::httpRequest()->post($url, $params); + + Logger::log('subscribe_to_hub: returns: ' . $postResult->getReturnCode(), Logger::DEBUG); + + return; + + } }