Only process feed and dfrn items when they aren't already stored

diff --git a/include/dfrn.php b/include/dfrn.php
index 702fbb15fe0f31a19c5eb831f4a0cb8d91b17f03..903db9b601f4e215e7b7cfff4f13aec2639664a2 100644 (file)
--- a/include/dfrn.php
+++ b/include/dfrn.php
@@ -391,7 +391,7 @@ class dfrn {
         *
         * @return object XML root object
         */
-       private function add_header($doc, $owner, $authorelement, $alternatelink = "", $public = false) {
+       private static function add_header($doc, $owner, $authorelement, $alternatelink = "", $public = false) {
 
                if ($alternatelink == "")
                        $alternatelink = $owner['url'];
@@ -462,7 +462,7 @@ class dfrn {
         *
         * @return object XML author object
         */
-       private function add_author($doc, $owner, $authorelement, $public) {
+       private static function add_author($doc, $owner, $authorelement, $public) {
 
                // Is the profile hidden or shouldn't be published in the net? Then add the "hide" element
                $r = q("SELECT `id` FROM `profile` INNER JOIN `user` ON `user`.`uid` = `profile`.`uid`
@@ -591,7 +591,7 @@ class dfrn {
         *
         * @return object XML author object
         */
-       private function add_entry_author($doc, $element, $contact_url, $item) {
+       private static function add_entry_author($doc, $element, $contact_url, $item) {
 
                $contact = get_contact_details_by_url($contact_url, $item["uid"]);
 
@@ -631,7 +631,7 @@ class dfrn {
         *
         * @return object XML activity object
         */
-       private function create_activity($doc, $element, $activity) {
+       private static function create_activity($doc, $element, $activity) {
 
                if($activity) {
                        $entry = $doc->createElement($element);
@@ -685,7 +685,7 @@ class dfrn {
         *
         * @return object XML attachment object
         */
-       private function get_attachment($doc, $root, $item) {
+       private static function get_attachment($doc, $root, $item) {
                $arr = explode('[/attach],',$item['attach']);
                if(count($arr)) {
                        foreach($arr as $r) {
@@ -720,7 +720,7 @@ class dfrn {
         *
         * @return object XML entry object
         */
-       private function entry($doc, $type, $item, $owner, $comment = false, $cid = 0) {
+       private static function entry($doc, $type, $item, $owner, $comment = false, $cid = 0) {
 
                $mentioned = array();
 
@@ -1103,7 +1103,7 @@ class dfrn {
         * @param string $birthday Birthday of the contact
         *
         */
-       private function birthday_event($contact, $birthday) {
+       private static function birthday_event($contact, $birthday) {
 
                // Check for duplicates
                $r = q("SELECT `id` FROM `event` WHERE `uid` = %d AND `cid` = %d AND `start` = '%s' AND `type` = '%s' LIMIT 1",
@@ -1146,7 +1146,7 @@ class dfrn {
         *
         * @return Returns an array with relevant data of the author
         */
-       private function fetchauthor($xpath, $context, $importer, $element, $onlyfetch, $xml = "") {
+       private static function fetchauthor($xpath, $context, $importer, $element, $onlyfetch, $xml = "") {
 
                $author = array();
                $author["name"] = $xpath->evaluate($element."/atom:name/text()", $context)->item(0)->nodeValue;
@@ -1358,7 +1358,7 @@ class dfrn {
         *
         * @return string XML string
         */
-       private function transform_activity($xpath, $activity, $element) {
+       private static function transform_activity($xpath, $activity, $element) {
                if (!is_object($activity))
                        return "";
 
@@ -1403,7 +1403,7 @@ class dfrn {
         * @param object $mail mail elements
         * @param array $importer Record of the importer user mixed with contact of the content
         */
-       private function process_mail($xpath, $mail, $importer) {
+       private static function process_mail($xpath, $mail, $importer) {
 
                logger("Processing mails");
 
@@ -1454,7 +1454,7 @@ class dfrn {
         * @param object $suggestion suggestion elements
         * @param array $importer Record of the importer user mixed with contact of the content
         */
-       private function process_suggestion($xpath, $suggestion, $importer) {
+       private static function process_suggestion($xpath, $suggestion, $importer) {
                $a = get_app();
 
                logger("Processing suggestions");
@@ -1556,7 +1556,7 @@ class dfrn {
         * @param object $relocation relocation elements
         * @param array $importer Record of the importer user mixed with contact of the content
         */
-       private function process_relocation($xpath, $relocation, $importer) {
+       private static function process_relocation($xpath, $relocation, $importer) {
 
                logger("Processing relocations");
 
@@ -1685,7 +1685,7 @@ class dfrn {
         * @param array $importer Record of the importer user mixed with contact of the content
         * @param int $entrytype Is it a toplevel entry, a comment or a relayed comment?
         */
-       private function update_content($current, $item, $importer, $entrytype) {
+       private static function update_content($current, $item, $importer, $entrytype) {
                $changed = false;
 
                if (edited_timestamp_is_newer($current, $item)) {
@@ -1737,7 +1737,7 @@ class dfrn {
         *
         * @return int Is it a toplevel entry, a comment or a relayed comment?
         */
-       private function get_entry_type($importer, $item) {
+       private static function get_entry_type($importer, $item) {
                if ($item["parent-uri"] != $item["uri"]) {
                        $community = false;
 
@@ -1803,7 +1803,7 @@ class dfrn {
         * @param array $importer Record of the importer user mixed with contact of the content
         * @param int $posted_id The record number of item record that was just posted
         */
-       private function do_poke($item, $importer, $posted_id) {
+       private static function do_poke($item, $importer, $posted_id) {
                $verb = urldecode(substr($item["verb"],strpos($item["verb"], "#")+1));
                if(!$verb)
                        return;
@@ -1858,7 +1858,7 @@ class dfrn {
         *
         * @return bool Should the processing of the entries be continued?
         */
-       private function process_verbs($entrytype, $importer, &$item, &$is_like) {
+       private static function process_verbs($entrytype, $importer, &$item, &$is_like) {
 
                logger("Process verb ".$item["verb"]." and object-type ".$item["object-type"]." for entrytype ".$entrytype, LOGGER_DEBUG);
 
@@ -1958,7 +1958,7 @@ class dfrn {
         * @param object $links link elements
         * @param array $item the item record
         */
-       private function parse_links($links, &$item) {
+       private static function parse_links($links, &$item) {
                $rel = "";
                $href = "";
                $type = "";
@@ -2001,7 +2001,7 @@ class dfrn {
         * @param object $entry entry elements
         * @param array $importer Record of the importer user mixed with contact of the content
         */
-       private function process_entry($header, $xpath, $entry, $importer) {
+       private static function process_entry($header, $xpath, $entry, $importer) {
 
                logger("Processing entries");
 
@@ -2010,6 +2010,20 @@ class dfrn {
                // Get the uri
                $item["uri"] = $xpath->query("atom:id/text()", $entry)->item(0)->nodeValue;
 
+               $item["edited"] = $xpath->query("atom:updated/text()", $entry)->item(0)->nodeValue;
+
+               $current = q("SELECT `id`, `uid`, `last-child`, `edited`, `body` FROM `item` WHERE `uri` = '%s' AND `uid` = %d LIMIT 1",
+                       dbesc($item["uri"]),
+                       intval($importer["importer_uid"])
+               );
+
+               // Is there an existing item?
+               if (dbm::is_result($current) AND edited_timestamp_is_newer($current[0], $item) AND
+                       (datetime_convert("UTC","UTC",$item["edited"]) < $current[0]["edited"])) {
+                       logger("Item ".$item["uri"]." already existed.", LOGGER_DEBUG);
+                       return;
+               }
+
                // Fetch the owner
                $owner = self::fetchauthor($xpath, $entry, $importer, "dfrn:owner", true);
 
@@ -2027,7 +2041,6 @@ class dfrn {
                $item["title"] = $xpath->query("atom:title/text()", $entry)->item(0)->nodeValue;
 
                $item["created"] = $xpath->query("atom:published/text()", $entry)->item(0)->nodeValue;
-               $item["edited"] = $xpath->query("atom:updated/text()", $entry)->item(0)->nodeValue;
 
                $item["body"] = $xpath->query("dfrn:env/text()", $entry)->item(0)->nodeValue;
                $item["body"] = str_replace(array(' ',"\t","\r","\n"), array('','','',''),$item["body"]);
@@ -2215,18 +2228,13 @@ class dfrn {
                        }
                }
 
-               $r = q("SELECT `id`, `uid`, `last-child`, `edited`, `body` FROM `item` WHERE `uri` = '%s' AND `uid` = %d LIMIT 1",
-                       dbesc($item["uri"]),
-                       intval($importer["importer_uid"])
-               );
-
                if (!self::process_verbs($entrytype, $importer, $item, $is_like)) {
                        logger("Exiting because 'process_verbs' told us so", LOGGER_DEBUG);
                        return;
                }
 
                // Update content if 'updated' changes
-               if (dbm::is_result($r)) {
+               if (dbm::is_result($current)) {
                        if (self::update_content($r[0], $item, $importer, $entrytype))
                                logger("Item ".$item["uri"]." was updated.", LOGGER_DEBUG);
                        else
@@ -2311,7 +2319,7 @@ class dfrn {
         * @param object $deletion deletion elements
         * @param array $importer Record of the importer user mixed with contact of the content
         */
-       private function process_deletion($xpath, $deletion, $importer) {
+       private static function process_deletion($xpath, $deletion, $importer) {
 
                logger("Processing deletions");
 

diff --git a/include/feed.php b/include/feed.php
index 579ff7caae3652cce6c9fcb714720886bd32334f..e0fef50dbbc6f7ed832e11451bf6bed47df3c9f6 100644 (file)
--- a/include/feed.php
+++ b/include/feed.php
@@ -177,18 +177,6 @@ function feed_import($xml,$importer,&$contact, &$hub, $simulate = false) {
        foreach (array_reverse($entrylist) AS $entry) {
                $item = array_merge($header, $author);
 
-               $item["title"] = $xpath->evaluate('atom:title/text()', $entry)->item(0)->nodeValue;
-
-               if ($item["title"] == "")
-                       $item["title"] = $xpath->evaluate('title/text()', $entry)->item(0)->nodeValue;
-
-               if ($item["title"] == "")
-                       $item["title"] = $xpath->evaluate('rss:title/text()', $entry)->item(0)->nodeValue;
-
-               $alternate = $xpath->query("atom:link[@rel='alternate']", $entry)->item(0)->attributes;
-               if (!is_object($alternate))
-                       $alternate = $xpath->query("atom:link", $entry)->item(0)->attributes;
-
                if (is_object($alternate))
                        foreach($alternate AS $attributes)
                                if ($attributes->name == "href")
@@ -212,6 +200,27 @@ function feed_import($xml,$importer,&$contact, &$hub, $simulate = false) {
 
                $item["parent-uri"] = $item["uri"];
 
+               if (!$simulate) {
+                       $r = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s', '%s')",
+                               intval($importer["uid"]), dbesc($item["uri"]), dbesc(NETWORK_FEED), dbesc(NETWORK_DFRN));
+                       if ($r) {
+                               logger("Item with uri ".$item["uri"]." for user ".$importer["uid"]." already existed under id ".$r[0]["id"], LOGGER_DEBUG);
+                               continue;
+                       }
+               }
+
+               $item["title"] = $xpath->evaluate('atom:title/text()', $entry)->item(0)->nodeValue;
+
+               if ($item["title"] == "")
+                       $item["title"] = $xpath->evaluate('title/text()', $entry)->item(0)->nodeValue;
+
+               if ($item["title"] == "")
+                       $item["title"] = $xpath->evaluate('rss:title/text()', $entry)->item(0)->nodeValue;
+
+               $alternate = $xpath->query("atom:link[@rel='alternate']", $entry)->item(0)->attributes;
+               if (!is_object($alternate))
+                       $alternate = $xpath->query("atom:link", $entry)->item(0)->attributes;
+
                $published = $xpath->query('atom:published/text()', $entry)->item(0)->nodeValue;
 
                if ($published == "")
@@ -250,15 +259,6 @@ function feed_import($xml,$importer,&$contact, &$hub, $simulate = false) {
                if ($creator != "")
                        $item["author-name"] = $creator;
 
-               if (!$simulate) {
-                       $r = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s', '%s')",
-                               intval($importer["uid"]), dbesc($item["uri"]), dbesc(NETWORK_FEED), dbesc(NETWORK_DFRN));
-                       if ($r) {
-                               logger("Item with uri ".$item["uri"]." for user ".$importer["uid"]." already existed under id ".$r[0]["id"], LOGGER_DEBUG);
-                               continue;
-                       }
-               }
-
                /// @TODO ?
                // <category>Ausland</category>
                // <media:thumbnail width="152" height="76" url="http://www.taz.de/picture/667875/192/14388767.jpg"/>

diff --git a/include/poller.php b/include/poller.php
index e8bfb88389aebcb08355d5792df373f89b6a9d07..16198580f3bc5851bb11b42c0b0d9e6f2a74eefd 100644 (file)
--- a/include/poller.php
+++ b/include/poller.php
@@ -206,6 +206,16 @@ function poller_exec_function($queue, $funcname, $argv) {
 
        $duration = number_format(microtime(true) - $stamp, 3);
 
+       if ($duration > 3600) {
+               logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 1 hour (".round($duration/60, 3).")", LOGGER_DEBUG);
+       } elseif ($duration > 600) {
+               logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 10 minutes (".round($duration/60, 3).")", LOGGER_DEBUG);
+       } elseif ($duration > 300) {
+               logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 5 minutes (".round($duration/60, 3).")", LOGGER_DEBUG);
+       } elseif ($duration > 120) {
+               logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 2 minutes (".round($duration/60, 3).")", LOGGER_DEBUG);
+       }
+
        logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds.");
 
        // Write down the performance values into the log