]> git.mxchange.org Git - friendica.git/commitdiff
Merge remote-tracking branch 'upstream/develop' into onepoll
authorMichael <heluecht@pirati.ca>
Sun, 3 Dec 2017 13:45:48 +0000 (13:45 +0000)
committerMichael <heluecht@pirati.ca>
Sun, 3 Dec 2017 13:45:48 +0000 (13:45 +0000)
1  2 
src/Worker/Cron.php
src/Worker/OnePoll.php

diff --combined src/Worker/Cron.php
index 1a7a5784dc253aba3654c5110912958aacd71ed4,dd4ca4da2a28a067b07f026dc2a63738717c813e..43c33103b3cf375f1798538cbc48d4b5c525686d
@@@ -153,104 -153,113 +153,103 @@@ Class Cron 
                        : ''
                );
  
 -              $contacts = q("SELECT `contact`.`id` FROM `user`
 +              $contacts = q("SELECT `contact`.`id`, `contact`.`nick`, `contact`.`name`, `contact`.`network`,
 +                                      `contact`.`last-update`, `contact`.`priority`, `contact`.`subhub`
 +                              FROM `user`
                                STRAIGHT_JOIN `contact`
                                ON `contact`.`uid` = `user`.`uid` AND `contact`.`rel` IN (%d, %d) AND `contact`.`poll` != ''
-                                       AND `contact`.`network` IN ('%s', '%s', '%s', '%s', '%s', '%s') $sql_extra
 -                                      AND `contact`.`network` IN ('%s', '%s', '%s', '%s') $sql_extra
++                                      AND `contact`.`network` IN ('%s', '%s', '%s', '%s', '%s') $sql_extra
                                        AND NOT `contact`.`self` AND NOT `contact`.`blocked` AND NOT `contact`.`readonly`
                                        AND NOT `contact`.`archive`
 -                              WHERE NOT `user`.`account_expired` AND NOT `user`.`account_removed` $abandon_sql ORDER BY RAND()",
 +                              WHERE NOT `user`.`account_expired` AND NOT `user`.`account_removed` $abandon_sql",
                        intval(CONTACT_IS_SHARING),
                        intval(CONTACT_IS_FRIEND),
                        dbesc(NETWORK_DFRN),
                        dbesc(NETWORK_OSTATUS),
 +                      dbesc(NETWORK_DIASPORA),
                        dbesc(NETWORK_FEED),
-                       dbesc(NETWORK_MAIL),
-                       dbesc(NETWORK_MAIL2)
+                       dbesc(NETWORK_MAIL)
                );
  
                if (!DBM::is_result($contacts)) {
                        return;
                }
  
 -              foreach ($contacts as $c) {
 +              foreach ($contacts as $contact) {
  
 -                      $res = q("SELECT * FROM `contact` WHERE `id` = %d LIMIT 1",
 -                              intval($c['id'])
 -                      );
 -
 -                      if (!DBM::is_result($res)) {
 -                              continue;
 +                      if ($manual_id) {
 +                              $contact['last-update'] = NULL_DATE;
                        }
  
 -                      foreach ($res as $contact) {
 -
 -                              $xml = false;
 -
 -                              if ($manual_id) {
 -                                      $contact['last-update'] = NULL_DATE;
 -                              }
 +                      if (in_array($contact['network'], array(NETWORK_DFRN, NETWORK_OSTATUS))) {
 +                              $contact['priority'] = 2;
 +                      }
  
 -                              if (in_array($contact['network'], array(NETWORK_DFRN, NETWORK_OSTATUS))) {
 -                                      $contact['priority'] = 2;
 -                              }
 +                      if ($contact['subhub'] && in_array($contact['network'], array(NETWORK_DFRN, NETWORK_OSTATUS))) {
 +                              /*
 +                               * We should be getting everything via a hub. But just to be sure, let's check once a day.
 +                               * (You can make this more or less frequent if desired by setting 'pushpoll_frequency' appropriately)
 +                               * This also lets us update our subscription to the hub, and add or replace hubs in case it
 +                               * changed. We will only update hubs once a day, regardless of 'pushpoll_frequency'.
 +                               */
 +                              $poll_interval = Config::get('system', 'pushpoll_frequency');
 +                              $contact['priority'] = (($poll_interval !== false) ? intval($poll_interval) : 3);
 +                      }
  
 -                              if ($contact['subhub'] && in_array($contact['network'], array(NETWORK_DFRN, NETWORK_OSTATUS))) {
 -                                      /*
 -                                       * We should be getting everything via a hub. But just to be sure, let's check once a day.
 -                                       * (You can make this more or less frequent if desired by setting 'pushpoll_frequency' appropriately)
 -                                       * This also lets us update our subscription to the hub, and add or replace hubs in case it
 -                                       * changed. We will only update hubs once a day, regardless of 'pushpoll_frequency'.
 -                                       */
 -                                      $poll_interval = Config::get('system', 'pushpoll_frequency');
 -                                      $contact['priority'] = (($poll_interval !== false) ? intval($poll_interval) : 3);
 +                      if (($contact['priority'] >= 0) && !$force) {
 +                              $update = false;
 +
 +                              $t = $contact['last-update'];
 +
 +                              /*
 +                               * Based on $contact['priority'], should we poll this site now? Or later?
 +                               */
 +                              switch ($contact['priority']) {
 +                                      case 5:
 +                                              if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 month")) {
 +                                                      $update = true;
 +                                              }
 +                                              break;
 +                                      case 4:
 +                                              if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 week")) {
 +                                                      $update = true;
 +                                              }
 +                                              break;
 +                                      case 3:
 +                                              if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 day")) {
 +                                                      $update = true;
 +                                              }
 +                                              break;
 +                                      case 2:
 +                                              if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 12 hour")) {
 +                                                      $update = true;
 +                                              }
 +                                              break;
 +                                      case 1:
 +                                              if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 hour")) {
 +                                                      $update = true;
 +                                              }
 +                                              break;
 +                                      case 0:
 +                                      default:
 +                                              if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + ".$min_poll_interval." minute")) {
 +                                                      $update = true;
 +                                              }
 +                                              break;
                                }
 -
 -                              if (($contact['priority'] >= 0) && !$force) {
 -                                      $update = false;
 -
 -                                      $t = $contact['last-update'];
 -
 -                                      /*
 -                                       * Based on $contact['priority'], should we poll this site now? Or later?
 -                                       */
 -                                      switch ($contact['priority']) {
 -                                              case 5:
 -                                                      if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 month")) {
 -                                                              $update = true;
 -                                                      }
 -                                                      break;
 -                                              case 4:
 -                                                      if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 week")) {
 -                                                              $update = true;
 -                                                      }
 -                                                      break;
 -                                              case 3:
 -                                                      if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 day")) {
 -                                                              $update = true;
 -                                                      }
 -                                                      break;
 -                                              case 2:
 -                                                      if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 12 hour")) {
 -                                                              $update = true;
 -                                                      }
 -                                                      break;
 -                                              case 1:
 -                                                      if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 hour")) {
 -                                                              $update = true;
 -                                                      }
 -                                                      break;
 -                                              case 0:
 -                                              default:
 -                                                      if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + ".$min_poll_interval." minute")) {
 -                                                              $update = true;
 -                                                      }
 -                                                      break;
 -                                      }
 -                                      if (!$update) {
 -                                              continue;
 -                                      }
 +                              if (!$update) {
 +                                      continue;
                                }
 +                      }
  
 -                              logger("Polling " . $contact["network"] . " " . $contact["id"] . " " . $contact["nick"] . " " . $contact["name"]);
 +                      logger("Polling " . $contact["network"] . " " . $contact["id"] . " " . $contact["nick"] . " " . $contact["name"]);
  
 -                              if (($contact['network'] == NETWORK_FEED) && ($contact['priority'] <= 3)) {
 -                                      $priority = PRIORITY_MEDIUM;
 -                              } else {
 -                                      $priority = PRIORITY_LOW;
 -                              }
 -                              Worker::add(array('priority' => $priority, 'dont_fork' => true), 'OnePoll', (int)$contact['id']);
 +                      if (($contact['network'] == NETWORK_FEED) && ($contact['priority'] <= 3)) {
 +                              $priority = PRIORITY_MEDIUM;
 +                      } else {
 +                              $priority = PRIORITY_LOW;
                        }
 +                      Worker::add(array('priority' => $priority, 'dont_fork' => true), 'OnePoll', (int)$contact['id']);
                }
        }
  }
diff --combined src/Worker/OnePoll.php
index c5a62e9826ca048408cf188e75f58333a71e7461,439997be6f0defa19dc1c993717198b9b4f0627f..64c533f8201768bb4b3ecf3a5c84cd07f4b5b56b
@@@ -23,7 -23,7 +23,7 @@@ Class OnePol
                require_once 'include/items.php';
                require_once 'include/queue_fn.php';
  
 -              logger('onepoll: start');
 +              logger('start');
  
                $manual_id  = 0;
                $generation = 0;
@@@ -36,7 -36,7 +36,7 @@@
                }
  
                if (!$contact_id) {
 -                      logger('onepoll: no contact');
 +                      logger('no contact');
                        return;
                }
  
                                $last_updated = PortableContact::lastUpdated($contact["url"]);
                                $updated = datetime_convert();
                                if ($last_updated) {
 +                                      logger('Diaspora contact '.$contact['id'].' had last update on '.$last_updated, LOGGER_DEBUG);
 +
 +                                      // The last public item can be older than the last item we got
 +                                      if ($last_updated < $contact['last-item']) {
 +                                              $last_updated = $contact['last-item'];
 +                                      }
 +
                                        $fields = array('last-item' => $last_updated, 'last-update' => $updated, 'success_update' => $updated);
                                        dba::update('contact', $fields, array('id' => $contact['id']));
 +                                      Contact::unmarkForArchival($contact);
                                } else {
                                        dba::update('contact', array('last-update' => $updated, 'failure_update' => $updated), array('id' => $contact['id']));
 +                                      Contact::markForArchival($contact);
 +                                      logger('Diaspora contact '.$contact['id'].' is marked for archival', LOGGER_DEBUG);
                                }
                        }
                        return;
                if (($contact['network'] === NETWORK_OSTATUS) || ($contact['network'] === NETWORK_DIASPORA) || ($contact['network'] === NETWORK_DFRN)) {
                        if (!PortableContact::reachable($contact['url'])) {
                                logger("Skipping probably dead contact ".$contact['url']);
 +
 +                              // set the last-update so we don't keep polling
 +                              dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
                                return;
                        }
  
                        if (!update_contact($contact["id"])) {
                                Contact::markForArchival($contact);
                                logger('Contact is marked dead');
 +
 +                              // set the last-update so we don't keep polling
 +                              dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
                                return;
                        } else {
                                Contact::unmarkForArchival($contact);
  
                if ($importer_uid == 0) {
                        logger('Ignore public contacts');
 +
 +                      // set the last-update so we don't keep polling
 +                      dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
                        return;
                }
  
  
                if (!DBM::is_result($r)) {
                        logger('No self contact for user '.$importer_uid);
 +
 +                      // set the last-update so we don't keep polling
 +                      dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
                        return;
                }
  
                $importer = $r[0];
  
 -              logger("onepoll: poll: ({$contact['id']}) IMPORTER: {$importer['name']}, CONTACT: {$contact['name']}");
 +              logger("poll: ({$contact['network']}-{$contact['id']}) IMPORTER: {$importer['name']}, CONTACT: {$contact['name']}");
  
                if ($contact['network'] === NETWORK_DFRN) {
                        $idtosend = $orig_id = (($contact['dfrn-id']) ? $contact['dfrn-id'] : $contact['issued-id']);
                        $ret = z_fetch_url($url);
  
                        if ($ret['errno'] == CURLE_OPERATION_TIMEDOUT) {
 +                              // set the last-update so we don't keep polling
 +                              dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
 +                              Contact::markForArchival($contact);
                                return;
                        }
  
  
                        $html_code = $a->get_curl_code();
  
 -                      logger('onepoll: handshake with url ' . $url . ' returns xml: ' . $handshake_xml, LOGGER_DATA);
 +                      logger('handshake with url ' . $url . ' returns xml: ' . $handshake_xml, LOGGER_DATA);
  
  
                        if (!strlen($handshake_xml) || ($html_code >= 400) || !$html_code) {
                                // set the last-update so we don't keep polling
                                $fields = array('last-update' => datetime_convert(), 'failure_update' => datetime_convert());
                                dba::update('contact', $fields, array('id' => $contact['id']));
 -
                                return;
                        }
  
  
                                $fields = array('last-update' => datetime_convert(), 'failure_update' => datetime_convert());
                                dba::update('contact', $fields, array('id' => $contact['id']));
 -
                                return;
                        }
  
                        }
  
                        if ((intval($res->status) != 0) || !strlen($res->challenge) || !strlen($res->dfrn_id)) {
 +                              // set the last-update so we don't keep polling
 +                              dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
                                return;
                        }
  
                        }
  
                        if ($final_dfrn_id != $orig_id) {
 -                              logger('ID did not decode: ' . $contact['id'] . ' orig: ' . $orig_id . ' final: ' . $final_dfrn_id);
                                // did not decode properly - cannot trust this site
 +                              logger('ID did not decode: ' . $contact['id'] . ' orig: ' . $orig_id . ' final: ' . $final_dfrn_id);
 +
 +                              // set the last-update so we don't keep polling
 +                              dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
 +                              Contact::markForArchival($contact);
                                return;
                        }
  
                        // Are we allowed to import from this person?
  
                        if ($contact['rel'] == CONTACT_IS_FOLLOWER || $contact['blocked'] || $contact['readonly']) {
 +                              // set the last-update so we don't keep polling
 +                              dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
                                return;
                        }
  
                        $ret = z_fetch_url($contact['poll'], false, $redirects, array('cookiejar' => $cookiejar));
  
                        if ($ret['errno'] == CURLE_OPERATION_TIMEDOUT) {
 +                              // set the last-update so we don't keep polling
 +                              dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
 +                              Contact::markForArchival($contact);
                                return;
                        }
  
                        $xml = $ret['body'];
  
                        unlink($cookiejar);
-               } elseif ($contact['network'] === NETWORK_MAIL || $contact['network'] === NETWORK_MAIL2) {
+               } elseif ($contact['network'] === NETWORK_MAIL) {
  
                        logger("Mail: Fetching for ".$contact['addr'], LOGGER_DEBUG);
  
                        $mail_disabled = ((function_exists('imap_open') && (! Config::get('system', 'imap_disabled'))) ? 0 : 1);
                        if ($mail_disabled) {
 +                              // set the last-update so we don't keep polling
 +                              dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
 +                              Contact::markForArchival($contact);
                                return;
                        }
  
  
                                        $metas = Email::messageMeta($mbox, implode(',', $msgs));
                                        if (count($metas) != count($msgs)) {
 -                                              logger("onepoll: for " . $mailconf['user'] . " there are ". count($msgs) . " messages but received " . count($metas) . " metas", LOGGER_DEBUG);
 +                                              logger("for " . $mailconf['user'] . " there are ". count($msgs) . " messages but received " . count($metas) . " metas", LOGGER_DEBUG);
                                        } else {
                                                $msgs = array_combine($msgs, $metas);
  
  
                                $fields = array('last-update' => datetime_convert(), 'failure_update' => datetime_convert());
                                dba::update('contact', $fields, array('id' => $contact['id']));
 -
 +                              Contact::markForArchival($contact);
                                return;
                        }