Speed up of OStatus transmission/Ignoring "created" with remote self

diff --git a/include/items.php b/include/items.php
index e30994d5f1a096068b6707a9eab8734ed664b48e..94401509b49853eae5f6adf038f15ee21fa3b0da 100644 (file)
--- a/include/items.php
+++ b/include/items.php
@@ -623,11 +623,11 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f
        $arr['owner-name']    = ((x($arr, 'owner-name'))    ? trim($arr['owner-name'])    : '');
        $arr['owner-link']    = ((x($arr, 'owner-link'))    ? notags(trim($arr['owner-link']))    : '');
        $arr['owner-avatar']  = ((x($arr, 'owner-avatar'))  ? notags(trim($arr['owner-avatar']))  : '');
-       $arr['created']       = ((x($arr, 'created') !== false) ? datetime_convert('UTC','UTC', $arr['created']) : datetime_convert());
-       $arr['edited']        = ((x($arr, 'edited')  !== false) ? datetime_convert('UTC','UTC', $arr['edited'])  : datetime_convert());
-       $arr['commented']     = ((x($arr, 'commented')  !== false) ? datetime_convert('UTC','UTC', $arr['commented'])  : datetime_convert());
-       $arr['received']      = ((x($arr, 'received')  !== false) ? datetime_convert('UTC','UTC', $arr['received'])  : datetime_convert());
-       $arr['changed']       = ((x($arr, 'changed')  !== false) ? datetime_convert('UTC','UTC', $arr['changed'])  : datetime_convert());
+       $arr['received']      = ((x($arr, 'received') !== false) ? datetime_convert('UTC','UTC', $arr['received']) : datetime_convert());
+       $arr['created']       = ((x($arr, 'created') !== false) ? datetime_convert('UTC','UTC', $arr['created']) : $arr['received']);
+       $arr['edited']        = ((x($arr, 'edited') !== false) ? datetime_convert('UTC','UTC', $arr['edited']) : $arr['created']);
+       $arr['changed']       = ((x($arr, 'changed') !== false) ? datetime_convert('UTC','UTC', $arr['changed']) : $arr['created']);
+       $arr['commented']     = ((x($arr, 'commented') !== false) ? datetime_convert('UTC','UTC', $arr['commented']) : $arr['created']);
        $arr['title']         = ((x($arr, 'title'))         ? trim($arr['title'])         : '');
        $arr['location']      = ((x($arr, 'location'))      ? trim($arr['location'])      : '');
        $arr['coord']         = ((x($arr, 'coord'))         ? notags(trim($arr['coord']))         : '');
@@ -1640,6 +1640,9 @@ function item_is_remote_self($contact, &$datarray) {
                        $datarray['author-name']   = $datarray['owner-name'];
                        $datarray['author-link']   = $datarray['owner-link'];
                        $datarray['author-avatar'] = $datarray['owner-avatar'];
+
+                       unset($datarray['created']);
+                       unset($datarray['edited']);
                }
 
                if ($contact['network'] != NETWORK_FEED) {

diff --git a/include/ostatus.php b/include/ostatus.php
index 16471f4d4c0010dc6e9a0861f5ab49510976b87b..6bec3987bb794ea4dfc3fb67682b749ed8e4d664 100644 (file)
--- a/include/ostatus.php
+++ b/include/ostatus.php
@@ -21,6 +21,7 @@ require_once 'include/follow.php';
 require_once 'include/api.php';
 require_once 'mod/proxy.php';
 require_once 'include/xml.php';
+require_once 'include/cache.php';
 
 /**
  * @brief This class contain functions for the OStatus protocol
@@ -195,7 +196,7 @@ class ostatus {
                        link_gcontact($gcid, $contact["uid"], $contact["id"]);
                }
 
-               return($author);
+               return $author;
        }
 
        /**
@@ -677,14 +678,14 @@ class ostatus {
                if ((count($elements) == 4) && ($elements[2] == "post"))
                        return "http://".$server[0]."/notice/".$elements[3];
 
-               if ((count($conversation) != 2) || ($conversation[1] ==""))
+               if ((count($conversation) != 2) || ($conversation[1] =="")) {
                        return $href;
-
-               if ($elements[3] == "objectType=thread")
+               }
+               if ($elements[3] == "objectType=thread") {
                        return "http://".$server[0]."/conversation/".$conversation[1];
-               else
+               } else {
                        return "http://".$server[0]."/notice/".$conversation[1];
-
+               }
                return $href;
        }
 
@@ -946,9 +947,9 @@ class ostatus {
                                STRAIGHT_JOIN `item` ON `item`.`parent` = `thritem`.`parent`
                                WHERE `term`.`uid` = %d AND `term`.`otype` = %d AND `term`.`type` = %d AND `term`.`url` = '%s'",
                                intval($uid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION), dbesc($conversation_url));
-               if ($parents)
+               if ($parents) {
                        $parent = $parents[0];
-               elseif (count($item) > 0) {
+               } elseif (count($item) > 0) {
                        $parent = $item;
                        $parent["type"] = "remote";
                        $parent["verb"] = ACTIVITY_POST;
@@ -956,9 +957,9 @@ class ostatus {
                } else {
                        // Preset the parent
                        $r = q("SELECT `id` FROM `contact` WHERE `self` AND `uid`=%d", $uid);
-                       if (!$r)
-                               return(-2);
-
+                       if (!dbm::is_result($r)) {
+                               return -2;
+                       }
                        $parent = array();
                        $parent["id"] = 0;
                        $parent["parent"] = 0;
@@ -985,9 +986,9 @@ class ostatus {
                        } elseif (!$conv_arr["success"] && (substr($conv, 0, 8) == "https://")) {
                                $conv = str_replace("https://", "http://", $conv);
                                $conv_as = fetch_url($conv."?page=".$pageno);
-                       } else
+                       } else {
                                $conv_as = $conv_arr["body"];
-
+                       }
                        $conv_as = str_replace(',"statusnet:notice_info":', ',"statusnet_notice_info":', $conv_as);
                        $conv_as = json_decode($conv_as);
 
@@ -1015,9 +1016,10 @@ class ostatus {
                                        self::store_conversation($item_id, $conversation_url);
                                }
 
-                               return($item_stored);
-                       } else
-                               return(-3);
+                               return $item_stored;
+                       } else {
+                               return -3;
+                       }
                }
 
                $items = array_reverse($items);
@@ -1153,7 +1155,7 @@ class ostatus {
                        }
 
                        if (is_array($single_conv->to))
-                               foreach($single_conv->to AS $to)
+                               foreach ($single_conv->to AS $to)
                                        if ($importer["nurl"] == normalise_link($to->id))
                                                $mention = true;
 
@@ -1197,17 +1199,17 @@ class ostatus {
                                $arr['conversation-uri'] = $single_conv->status_net->conversation;
                        }
 
-                       if (isset($single_conv->status_net->notice_info->source))
+                       if (isset($single_conv->status_net->notice_info->source)) {
                                $arr["app"] = strip_tags($single_conv->status_net->notice_info->source);
-                       elseif (isset($single_conv->statusnet->notice_info->source))
+                       } elseif (isset($single_conv->statusnet->notice_info->source)) {
                                $arr["app"] = strip_tags($single_conv->statusnet->notice_info->source);
-                       elseif (isset($single_conv->statusnet_notice_info->source))
+                       } elseif (isset($single_conv->statusnet_notice_info->source)) {
                                $arr["app"] = strip_tags($single_conv->statusnet_notice_info->source);
-                       elseif (isset($single_conv->provider->displayName))
+                       } elseif (isset($single_conv->provider->displayName)) {
                                $arr["app"] = $single_conv->provider->displayName;
-                       else
+                       } else {
                                $arr["app"] = "OStatus";
-
+                       }
 
                        $arr["source"] = json_encode($single_conv);
                        $arr["protocol"] = PROTOCOL_GS_CONVERSATION;
@@ -1324,7 +1326,7 @@ class ostatus {
                        }
                }
 
-               return($item_stored);
+               return $item_stored;
        }
 
        /**
@@ -1367,16 +1369,16 @@ class ostatus {
                // Skip if it isn't a pure repeated messages
                // Does it start with a share?
                if (strpos($body, "[share") > 0)
-                       return("");
+                       return "";
 
                // Does it end with a share?
                if (strlen($body) > (strrpos($body, "[/share]") + 8))
-                       return("");
+                       return "";
 
                $attributes = preg_replace("/\[share(.*?)\]\s?(.*?)\s?\[\/share\]\s?/ism","$1",$body);
                // Skip if there is no shared message in there
                if ($body == $attributes)
-                       return(false);
+                       return false;
 
                $guid = "";
                preg_match("/guid='(.*?)'/ism", $attributes, $matches);
@@ -1496,12 +1498,12 @@ class ostatus {
                $hub = get_config('system','huburl');
 
                $hubxml = '';
-               if(strlen($hub)) {
+               if (strlen($hub)) {
                        $hubs = explode(',', $hub);
-                       if(count($hubs)) {
-                               foreach($hubs as $h) {
+                       if (count($hubs)) {
+                               foreach ($hubs as $h) {
                                        $h = trim($h);
-                                       if(! strlen($h))
+                                       if (! strlen($h))
                                                continue;
                                        if ($h === '[internal]')
                                                $h = App::get_baseurl() . '/pubsubhubbub';
@@ -1543,7 +1545,7 @@ class ostatus {
                                break;
                }
 
-               if (($siteinfo["type"] != "photo") && isset($siteinfo["image"])) {
+               if (!Config::get('system', 'ostatus_not_attach_preview') && ($siteinfo["type"] != "photo") && isset($siteinfo["image"])) {
                        $imgdata = get_photo_info($siteinfo["image"]);
                        $attributes = array("rel" => "enclosure",
                                        "href" => $siteinfo["image"],
@@ -1586,9 +1588,9 @@ class ostatus {
        private function add_author($doc, $owner) {
 
                $r = q("SELECT `homepage`, `publish` FROM `profile` WHERE `uid` = %d AND `is-default` LIMIT 1", intval($owner["uid"]));
-               if ($r)
+               if (dbm::is_result($r)) {
                        $profile = $r[0];
-
+               }
                $author = $doc->createElement("author");
                xml::add_element($doc, $author, "id", $owner["url"]);
                xml::add_element($doc, $author, "activity:object-type", ACTIVITY_OBJ_PERSON);
@@ -1743,22 +1745,22 @@ class ostatus {
 
                $r = q("SELECT * FROM `contact` WHERE `nurl` = '%s' AND `uid` IN (0, %d) ORDER BY `uid` DESC LIMIT 1",
                        dbesc(normalise_link($url)), intval($owner["uid"]));
-               if ($r) {
+               if (dbm::is_result($r)) {
                        $contact = $r[0];
                        $contact["uid"] = -1;
                }
 
-               if (!$r) {
+               if (!dbm::is_result($r)) {
                        $r = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1",
                                dbesc(normalise_link($url)));
-                       if ($r) {
+                       if (dbm::is_result($r)) {
                                $contact = $r[0];
                                $contact["uid"] = -1;
                                $contact["success_update"] = $contact["updated"];
                        }
                }
 
-               if (!$r)
+               if (!dbm::is_result($r))
                        $contact = owner;
 
                if (!isset($contact["poll"])) {
@@ -1797,11 +1799,11 @@ class ostatus {
                $r = q("SELECT * FROM `item` WHERE `uid` = %d AND `guid` = '%s' AND NOT `private` AND `network` IN ('%s', '%s', '%s') LIMIT 1",
                        intval($owner["uid"]), dbesc($repeated_guid),
                        dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS));
-               if ($r)
+               if (dbm::is_result($r)) {
                        $repeated_item = $r[0];
-               else
+               } else {
                        return false;
-
+               }
                $contact = self::contact_entry($repeated_item['author-link'], $owner);
 
                $parent_item = (($item['thr-parent']) ? $item['thr-parent'] : $item['parent-uri']);
@@ -2158,8 +2160,8 @@ class ostatus {
 
                $tags = item_getfeedtags($item);
 
-               if(count($tags))
-                       foreach($tags as $t)
+               if (count($tags))
+                       foreach ($tags as $t)
                                if ($t[0] == "@")
                                        $mentioned[$t[1]] = $t[1];
 
@@ -2194,8 +2196,8 @@ class ostatus {
                        xml::add_element($doc, $entry, "mastodon:scope", "public");
                }
 
-               if(count($tags))
-                       foreach($tags as $t)
+               if (count($tags))
+                       foreach ($tags as $t)
                                if ($t[0] != "@")
                                        xml::add_element($doc, $entry, "category", "", array("term" => $t[2]));
 
@@ -2227,19 +2229,33 @@ class ostatus {
         *
         * @return string XML feed
         */
-       public static function feed(App $a, $owner_nick, $last_update) {
+       public static function feed(App $a, $owner_nick, &$last_update) {
+               $stamp = microtime(true);
+
+               $cachekey = "ostatus:feed:".$owner_nick.":".$last_update;
+
+               $previous_created = $last_update;
+
+               $result = Cache::get($cachekey);
+               if (!is_null($result)) {
+                       logger('Feed duration: '.number_format(microtime(true) - $stamp, 3).' - '.$owner_nick.' - '.$previous_created.' (cached)', LOGGER_DEBUG);
+                       $last_update = $result['last_update'];
+                       return $result['feed'];
+               }
 
                $r = q("SELECT `contact`.*, `user`.`nickname`, `user`.`timezone`, `user`.`page-flags`
                                FROM `contact` INNER JOIN `user` ON `user`.`uid` = `contact`.`uid`
                                WHERE `contact`.`self` AND `user`.`nickname` = '%s' LIMIT 1",
                                dbesc($owner_nick));
-               if (!$r)
+               if (!dbm::is_result($r)) {
                        return;
+               }
 
                $owner = $r[0];
 
-               if (!strlen($last_update))
+               if (!strlen($last_update)) {
                        $last_update = 'now -30 days';
+               }
 
                $check_date = datetime_convert('UTC','UTC',$last_update,'Y-m-d H:i:s');
                $authorid = get_contact($owner["url"], 0);
@@ -2255,28 +2271,6 @@ class ostatus {
                                intval($authorid), dbesc($check_date),
                                dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN));
 
-/*             2016-10-23: The old query will be kept until we are sure that the query above is a good and fast replacement
-
-               $items = q("SELECT `item`.*, `item`.`id` AS `item_id` FROM `item`
-                               STRAIGHT_JOIN `thread` ON `thread`.`iid` = `item`.`parent`
-                               LEFT JOIN `item` AS `thritem` ON `thritem`.`uri`=`item`.`thr-parent` AND `thritem`.`uid`=`item`.`uid`
-                               WHERE `item`.`uid` = %d AND `item`.`received` > '%s' AND NOT `item`.`private` AND NOT `item`.`deleted`
-                                       AND `item`.`allow_cid` = '' AND `item`.`allow_gid` = '' AND `item`.`deny_cid`  = '' AND `item`.`deny_gid`  = ''
-                                       AND ((`item`.`wall` AND (`item`.`parent` = `item`.`id`))
-                                               OR (`item`.`network` = '%s' AND ((`thread`.`network` IN ('%s', '%s')) OR (`thritem`.`network` IN ('%s', '%s')))) AND `thread`.`mention`)
-                                       AND ((`item`.`owner-link` IN ('%s', '%s') AND (`item`.`parent` = `item`.`id`))
-                                               OR (`item`.`author-link` IN ('%s', '%s')))
-                               ORDER BY `item`.`id` DESC
-                               LIMIT 0, 300",
-                               intval($owner["uid"]), dbesc($check_date), dbesc(NETWORK_DFRN),
-                               //dbesc(NETWORK_OSTATUS), dbesc(NETWORK_OSTATUS),
-                               //dbesc(NETWORK_OSTATUS), dbesc(NETWORK_OSTATUS),
-                               dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN),
-                               dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN),
-                               dbesc($owner["nurl"]), dbesc(str_replace("http://", "https://", $owner["nurl"])),
-                               dbesc($owner["nurl"]), dbesc(str_replace("http://", "https://", $owner["nurl"]))
-                       );
-*/
                $doc = new DOMDocument('1.0', 'utf-8');
                $doc->formatOutput = true;
 
@@ -2288,9 +2282,20 @@ class ostatus {
                        }
                        $entry = self::entry($doc, $item, $owner);
                        $root->appendChild($entry);
+
+                       if ($last_update < $item['created']) {
+                               $last_update = $item['created'];
+                       }
                }
 
-               return(trim($doc->saveXML()));
+               $feeddata = trim($doc->saveXML());
+
+               $msg = array('feed' => $feeddata, 'last_update' => $last_update);
+               Cache::set($cachekey, $msg, CACHE_QUARTER_HOUR);
+
+               logger('Feed duration: '.number_format(microtime(true) - $stamp, 3).' - '.$owner_nick.' - '.$previous_created, LOGGER_DEBUG);
+
+               return $feeddata;
        }
 
        /**
@@ -2314,6 +2319,6 @@ class ostatus {
 
                $doc->appendChild($entry);
 
-               return(trim($doc->saveXML()));
+               return trim($doc->saveXML());
        }
 }

diff --git a/include/poller.php b/include/poller.php
index 89a17fa0c736fcd771a7d067205db5eb44b082cf..4831ff074743eef3e87ed8dd2317763522574eff 100644 (file)
--- a/include/poller.php
+++ b/include/poller.php
@@ -273,7 +273,9 @@ function poller_exec_function($queue, $funcname, $argv) {
 
        $argc = count($argv);
 
-       logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." ".$queue["parameter"]);
+       $new_process_id = uniqid("wrk", true);
+
+       logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." ".$queue["parameter"]." - Process PID: ".$new_process_id);
 
        $stamp = (float)microtime(true);
 
@@ -295,7 +297,7 @@ function poller_exec_function($queue, $funcname, $argv) {
        // For better logging create a new process id for every worker call
        // But preserve the old one for the worker
        $old_process_id = $a->process_id;
-       $a->process_id = uniqid("wrk", true);
+       $a->process_id = $new_process_id;
        $a->queue = $queue;
 
        $up_duration = number_format(microtime(true) - $poller_up_start, 3);
@@ -330,7 +332,7 @@ function poller_exec_function($queue, $funcname, $argv) {
                logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 2 minutes (".round($duration/60, 3).")", LOGGER_DEBUG);
        }
 
-       logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds.");
+       logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds. Process PID: ".$new_process_id);
 
        // Write down the performance values into the log
        if (Config::get("system", "profiler")) {

diff --git a/include/pubsubpublish.php b/include/pubsubpublish.php
index 580e3ffce18038f8b86f521f28acdf2d54a80aaf..2527963d4c786ec6029d90dbdd7197375ff424b6 100644 (file)
--- a/include/pubsubpublish.php
+++ b/include/pubsubpublish.php
@@ -14,7 +14,7 @@ function pubsubpublish_run(&$argv, &$argc){
        } else {
                // We'll push to each subscriber that has push > 0,
                // i.e. there has been an update (set in notifier.php).
-               $r = q("SELECT `id`, `callback_url` FROM `push_subscriber` WHERE `push` > 0");
+               $r = q("SELECT `id`, `callback_url` FROM `push_subscriber` WHERE `push` > 0 ORDER BY `last_update` DESC");
 
                foreach ($r as $rr) {
                        logger("Publish feed to ".$rr["callback_url"], LOGGER_DEBUG);
@@ -43,7 +43,13 @@ function handle_pubsubhubbub($id) {
 
        logger("Generate feed of user ".$rr['nickname']." to ".$rr['callback_url']." - last updated ".$rr['last_update'], LOGGER_DEBUG);
 
-       $params = ostatus::feed($a, $rr['nickname'], $rr['last_update']);
+       $last_update = $rr['last_update'];
+       $params = ostatus::feed($a, $rr['nickname'], $last_update);
+
+       if (!$params) {
+               return;
+       }
+
        $hmac_sig = hash_hmac("sha1", $params, $rr['secret']);
 
        $headers = array("Content-type: application/atom+xml",
@@ -60,10 +66,9 @@ function handle_pubsubhubbub($id) {
        if ($ret >= 200 && $ret <= 299) {
                logger('successfully pushed to '.$rr['callback_url']);
 
-               // set last_update to "now", and reset push=0
-               $date_now = datetime_convert('UTC','UTC','now','Y-m-d H:i:s');
+               // set last_update to the "created" date of the last item, and reset push=0
                q("UPDATE `push_subscriber` SET `push` = 0, last_update = '%s' WHERE id = %d",
-                       dbesc($date_now),
+                       dbesc($last_update),
                        intval($rr['id']));
 
        } else {