From 6fbb03801c7a5a3c6daaab52d603dffbadd8ff98 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 20 Jun 2015 08:44:29 +0200 Subject: [PATCH] The OStatus completion is now called before the original item is stored. --- include/items.php | 5 +- include/ostatus.php | 33 +-- include/ostatus_conversation.php | 343 +++++++++++++++++++++---------- 3 files changed, 259 insertions(+), 122 deletions(-) diff --git a/include/items.php b/include/items.php index fa370c82a8..b085ac8cfa 100644 --- a/include/items.php +++ b/include/items.php @@ -1192,12 +1192,15 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa // If there is no guid then take the same guid that was taken before for the same plink if ((trim($arr['guid']) == "") AND (trim($arr['plink']) != "") AND (trim($arr['network']) != "")) { logger('item_store: checking for an existing guid for plink '.$arr['plink'], LOGGER_DEBUG); - $r = q("SELECT `guid` FROM `guid` WHERE `plink` = '%s' AND `network` = '%s' LIMIT 1", + $r = q("SELECT `guid`, `uri` FROM `guid` WHERE `plink` = '%s' AND `network` = '%s' LIMIT 1", dbesc(trim($arr['plink'])), dbesc(trim($arr['network']))); if(count($r)) { $arr['guid'] = $r[0]["guid"]; logger('item_store: found guid '.$arr['guid'].' for plink '.$arr['plink'], LOGGER_DEBUG); + + if ($r[0]["uri"] != $arr['uri']) + logger('Different uri for same guid: '.$arr['uri'].' and '.$r[0]["uri"].' - this shouldnt happen!', LOGGER_DEBUG); } } diff --git a/include/ostatus.php b/include/ostatus.php index 86d0e36dbe..996cbea1fd 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -186,6 +186,7 @@ function ostatus_import($xml,$importer,&$contact, &$hub) { $item["body"] = add_page_info_to_body(html2bbcode($xpath->query('atom:content/text()', $entry)->item(0)->nodeValue)); $item["object-type"] = $xpath->query('activity:object-type/text()', $entry)->item(0)->nodeValue; + $item["object"] = $xml; $item["verb"] = $xpath->query('activity:verb/text()', $entry)->item(0)->nodeValue; if ($item["verb"] == ACTIVITY_FOLLOW) { @@ -218,6 +219,9 @@ function ostatus_import($xml,$importer,&$contact, &$hub) { if ($georsspoint) $item["coord"] = $georsspoint->item(0)->nodeValue; + // To-Do + // $item["location"] = + $categories = $xpath->query('atom:category', $entry); if ($categories) { foreach ($categories AS $category) { @@ -292,7 +296,7 @@ function ostatus_import($xml,$importer,&$contact, &$hub) { $repeat_of = ""; $notice_info = $xpath->query('statusnet:notice_info', $entry); - if ($notice_info) + if ($notice_info AND ($notice_info->length > 0)) { foreach($notice_info->item(0)->attributes AS $attributes) { if ($attributes->name == "source") $item["app"] = strip_tags($attributes->textContent); @@ -301,6 +305,7 @@ function ostatus_import($xml,$importer,&$contact, &$hub) { if ($attributes->name == "repeat_of") $repeat_of = $attributes->textContent; } + } // Is it a repeated post? if ($repeat_of != "") { @@ -376,12 +381,19 @@ function ostatus_import($xml,$importer,&$contact, &$hub) { } else $item["parent-uri"] = $item["uri"]; - // We risk the chance of getting orphan items, we correct it some lines later - // To-Do: See To-Do line below. - $item_id = item_store($item, true); + $item_id = ostatus_completion($conversation, $importer["uid"], $item); + + if ($item_id <= 0) { + $reason = $item_id; + $item["app"] .= $item_id; + $item_id = item_store($item, true); + if ($item_id) { + logger("Shouldn't happen. Code ".$reason." - uri ".$item["uri"], LOGGER_DEBUG); + complete_conversation($item_id, $conversation_url, true); + } + } //echo $xml; //print_r($item); - //echo $item_id." ".$item["parent-uri"]."\n"; if (!$item_id) { logger("Error storing item ".print_r($item, true), LOGGER_DEBUG); @@ -414,16 +426,5 @@ function ostatus_import($xml,$importer,&$contact, &$hub) { 'parent' => $item["parent"] )); } - - if ($conversation != "") { - // Check for duplicates. We really don't need to check the same conversation twice. - if (!in_array($conversation, $conversationlist)) { - // To-Do: - // Call this before item_store is called to avoid posts with orphans - // The routine then needs to get the item array. - complete_conversation($item_id, $conversation); - $conversationlist[] = $conversation; - } - } } } diff --git a/include/ostatus_conversation.php b/include/ostatus_conversation.php index e61f5fab4f..fd90ba3dd1 100644 --- a/include/ostatus_conversation.php +++ b/include/ostatus_conversation.php @@ -1,6 +1,8 @@ time()) { - logger('poll interval not reached'); - return; - } - } + if ($last AND !$override) { + $next = $last + ($poll_interval * 60); + if ($next > time()) { + logger('poll interval not reached'); + return; + } + } - logger('cron_start'); + logger('cron_start'); - $start = date("Y-m-d H:i:s", time() - ($poll_timeframe * 60)); - $conversations = q("SELECT `oid`, `url` FROM `term` WHERE `type` = 7 AND `term` > '%s' GROUP BY `url` ORDER BY `term` DESC", - dbesc($start)); + $start = date("Y-m-d H:i:s", time() - ($poll_timeframe * 60)); + $conversations = q("SELECT `oid`, `url`, `uid` FROM `term` WHERE `type` = 7 AND `term` > '%s' GROUP BY `url`, `uid` ORDER BY `term` DESC", + dbesc($start)); - foreach ($conversations AS $conversation) { - $id = $conversation['oid']; - $url = $conversation['url']; - complete_conversation($id, $url); - } + foreach ($conversations AS $conversation) { + ostatus_completion($conversation['url'], $conversation['uid']); + } - logger('cron_end'); + logger('cron_end'); - set_config('system','ostatus_last_poll', time()); + set_config('system','ostatus_last_poll', time()); } -function complete_conversation($itemid, $conversation_url, $only_add_conversation = false) { - global $a; +function ostatus_completion($conversation_url, $uid, $item = array()) { + + $item_stored = -3; $conversation_url = ostatus_convert_href($conversation_url); +/* +To-Do: if (intval(get_config('system','ostatus_poll_interval')) == -2) return; @@ -81,43 +84,56 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio $a->last_ostatus_conversation_url = $conversation_url; - $messages = q("SELECT `uid`, `parent`, `created`, `received`, `guid` FROM `item` WHERE `id` = %d LIMIT 1", intval($itemid)); - if (!$messages) - return; - $message = $messages[0]; - - // Store conversation url if not done before - $conversation = q("SELECT `url` FROM `term` WHERE `uid` = %d AND `oid` = %d AND `otype` = %d AND `type` = %d", - intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION)); - - if (!$conversation) { - $r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`, `created`, `received`, `guid`) VALUES (%d, %d, %d, %d, '%s', '%s', '%s', '%s', '%s')", - intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION), - dbesc($message["created"]), dbesc($conversation_url), dbesc($message["created"]), dbesc($message["received"]), dbesc($message["guid"])); - logger('Storing conversation url '.$conversation_url.' for id '.$itemid); - } - - if ($only_add_conversation) - return; +*/ // Get the parent - $parents = q("SELECT `id`, `uri`, `contact-id`, `type`, `verb`, `visible` FROM `item` WHERE `uid` = %d AND `id` = %d LIMIT 1", - intval($message["uid"]), intval($message["parent"])); - if (!$parents) - return; - $parent = $parents[0]; - - require_once('include/html2bbcode.php'); - require_once('include/items.php'); + $parents = q("SELECT `id`, `parent`, `uri`, `contact-id`, `type`, `verb`, `visible` FROM `item` WHERE `id` IN + (SELECT `parent` FROM `item` WHERE `id` IN + (SELECT `oid` FROM `term` WHERE `uid` = %d AND `otype` = %d AND `type` = %d AND `url` = '%s'))", + intval($uid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION), dbesc($conversation_url)); + + if ($parents) + $parent = $parents[0]; + elseif (count($item) > 0) { + $parent = $item; + $parent["type"] = "remote"; + $parent["verb"] = ACTIVITY_POST; + $parent["visible"] = 1; + } else { + // Preset the parent + $r = q("SELECT `id` FROM `contact` WHERE `self` AND `uid`=%d", $uid); + if (!$r) + return(-1); + + $parent = array(); + $parent["id"] = 0; + $parent["parent"] = 0; + $parent["uri"] = ""; + $parent["contact-id"] = $r[0]["id"]; + $parent["type"] = "remote"; + $parent["verb"] = ACTIVITY_POST; + $parent["visible"] = 1; + } $conv = str_replace("/conversation/", "/api/statusnet/conversation/", $conversation_url).".as"; $pageno = 1; $items = array(); - logger('fetching conversation url '.$conv.' for id '.$itemid.' and parent '.$parent["id"]); + logger('fetching conversation url '.$conv.' for user '.$uid); do { - $conv_as = fetch_url($conv."?page=".$pageno); + $conv_arr = z_fetch_url($conv."?page=".$pageno); + + // If it is a non-ssl site and there is an error, then try ssl or vice versa + if (!$conv_arr["success"] AND (substr($conv, 0, 7) == "http://")) { + $conv = str_replace("http://", "https://", $conv); + $conv_as = fetch_url($conv."?page=".$pageno); + } elseif (!$conv_arr["success"] AND (substr($conv, 0, 8) == "https://")) { + $conv = str_replace("https://", "http://", $conv); + $conv_as = fetch_url($conv."?page=".$pageno); + } else + $conv_as = $conv_arr["body"]; + $conv_as = str_replace(',"statusnet:notice_info":', ',"statusnet_notice_info":', $conv_as); $conv_as = json_decode($conv_as); @@ -130,69 +146,102 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio } while (true); - if (!sizeof($items)) - return; + logger('fetching conversation done. Found '.count($items).' items'); + + if (!sizeof($items)) { + if (count($item) > 0) { + $item_stored = item_store($item, true); + logger("Conversation ".$conversation_url." couldn't be fetched. Item uri ".$item["uri"]." stored: ".$item_stored, LOGGER_DEBUG); + + if ($item_stored) + complete_conversation($item_id, $conversation_url, true); + + return($item_stored); + } else + return(-2); + } $items = array_reverse($items); foreach ($items as $single_conv) { + + // Test - remove before flight + //$tempfile = tempnam(get_temppath(), "conversation"); + //file_put_contents($tempfile, json_encode($single_conv)); + + if (isset($single_conv->object->id)) $single_conv->id = $single_conv->object->id; - //logger("Got id ".$single_conv->id, LOGGER_DEBUG); - $plink = ostatus_convert_href($single_conv->id); if (isset($single_conv->object->url)) $plink = ostatus_convert_href($single_conv->object->url); - //logger("Got url ".$plink, LOGGER_DEBUG); - if (@!$single_conv->id) continue; + logger("Got id ".$single_conv->id, LOGGER_DEBUG); + if ($first_id == "") { $first_id = $single_conv->id; - $new_parents = q("SELECT `id`, `parent`, `uri`, `contact-id`, `type`, `verb`, `visible` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s','%s') LIMIT 1", - intval($message["uid"]), dbesc($first_id), - dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN)); - if ($new_parents) { - // It can happen that OStatus servers have incomplete threads. - // Then keep the current parent - if ($parent["id"] == $parent["parent"]) { - $parent = $new_parents[0]; - - if ($parent["id"] != $message["parent"]) - logger('Fetch new parent id '.$parent["id"].' - Old parent: '.$message["parent"]); + // The first post of the conversation isn't our first post. There are three options: + // 1. Our conversation hasn't the "real" thread starter + // 2. This first post is a post inside our thread + // 3. This first post is a post inside another thread + if (($first_id != $parent["uri"]) AND ($parent["uri"] != "")) { + $new_parents = q("SELECT `id`, `parent`, `uri`, `contact-id`, `type`, `verb`, `visible` FROM `item` WHERE `id` IN + (SELECT `parent` FROM `item` + WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s','%s')) LIMIT 1", + intval($uid), dbesc($first_id), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN)); + if ($new_parents) { + if ($new_parents[0]["parent"] == $parent["parent"]) { + // Option 2: This post is already present inside our thread - but not as thread starter + logger("Option 2: uri present in our thread: ".$first_id, LOGGER_DEBUG); + $first_id = $parent["uri"]; + } else { + // Option 3: Not so good. We have mixed parents. We have to see how to clean this up. + // For now just take the new parent. + $parent = $new_parents[0]; + $first_id = $parent["uri"]; + logger("Option 3: mixed parents for uri ".$first_id, LOGGER_DEBUG); + } } else { - $first_id = $parent["uri"]; - //logger('Keep parent for '.$itemid.' - Old parent: '.$message["parent"]); + // Option 1: We hadn't got the real thread starter + // We have to clean up our existing messages. + $parent["id"] = 0; + $parent["uri"] = $first_id; + logger("Option 1: we have a new parent: ".$first_id, LOGGER_DEBUG); } - } else { + } elseif ($parent["uri"] == "") { $parent["id"] = 0; $parent["uri"] = $first_id; } } - if (isset($single_conv->context->inReplyTo->id)) + if (isset($single_conv->context->inReplyTo->id)) { $parent_uri = $single_conv->context->inReplyTo->id; - else - $parent_uri = $parent["uri"]; - - $message_exists = q("SELECT `id`, `parent` FROM `item` WHERE `uid` = %d AND `plink` = '%s' AND `network` IN ('%s','%s') LIMIT 1", - intval($message["uid"]), dbesc($plink), - dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN)); - if (!$message_exists) - $message_exists = q("SELECT `id`, `parent` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s','%s') LIMIT 1", - intval($message["uid"]), dbesc($single_conv->id), - dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN)); + $parent_exists = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s','%s') LIMIT 1", + intval($uid), dbesc($parent_uri), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN)); + if (!$parent_exists) { + logger("Parent ".$parent_uri." wasn't found here", LOGGER_DEBUG); + $parent_uri = $parent["uri"]; + } + } else + $parent_uri = $parent["uri"]; + $message_exists = q("SELECT `id`, `parent`, `uri` FROM `item` WHERE `uid` = %d AND `uri` = '%s' AND `network` IN ('%s','%s') LIMIT 1", + intval($uid), dbesc($single_conv->id), + dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN)); if ($message_exists) { + logger("Message ".$single_conv->id." already existed on the system", LOGGER_DEBUG); + if ($parent["id"] != 0) { $existing_message = $message_exists[0]; // We improved the way we fetch OStatus messages, this shouldn't happen very often now + // To-Do: we have to change the shadow copies as well. This way here is really ugly. if ($existing_message["parent"] != $parent["id"]) { logger('updating id '.$existing_message["id"].' with parent '.$existing_message["parent"].' to parent '.$parent["id"].' uri '.$parent["uri"].' thread '.$parent_uri, LOGGER_DEBUG); @@ -216,6 +265,13 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio delete_thread($existing_message["parent"]); } } + + // The item we are having on the system is the one that we wanted to store via the item array + if (isset($item["uri"]) AND ($item["uri"] == $existing_message["uri"])) { + $item = array(); + $item_stored = 0; + } + continue; } @@ -224,7 +280,7 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio $actor = $single_conv->actor->url; $contact = q("SELECT `id` FROM `contact` WHERE `uid` = %d AND `nurl` = '%s' AND `network` != '%s'", - $message["uid"], normalise_link($actor), NETWORK_STATUSNET); + $uid, normalise_link($actor), NETWORK_STATUSNET); if (count($contact)) { logger("Found contact for url ".$actor, LOGGER_DEBUG); @@ -245,29 +301,19 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio $arr["network"] = NETWORK_OSTATUS; $arr["uri"] = $single_conv->id; $arr["plink"] = $plink; - $arr["uid"] = $message["uid"]; + $arr["uid"] = $uid; $arr["contact-id"] = $contact_id; - if ($parent["id"] != 0) - $arr["parent"] = $parent["id"]; - $arr["parent-uri"] = $parent["uri"]; - $arr["thr-parent"] = $parent_uri; + $arr["parent-uri"] = $parent_uri; $arr["created"] = $single_conv->published; $arr["edited"] = $single_conv->published; - $arr["owner-name"] = $single_conv->actor->contact->displayName; - //$arr["owner-name"] = $single_conv->actor->contact->preferredUsername; + $arr["owner-name"] = $single_conv->actor->displayName; if ($arr["owner-name"] == '') - $arr["owner-name"] = $single_conv->actor->portablecontacts_net->displayName; - if ($arr["owner-name"] == '') - $arr["owner-name"] = $single_conv->actor->displayName; - if ($arr["owner-name"] == '') - $arr["owner-name"] = $single_conv->actor->portablecontacts_net->preferredUsername; + $arr["owner-name"] = $single_conv->actor->contact->displayName; if ($arr["owner-name"] == '') - $arr["owner-name"] = $single_conv->actor->preferredUsername; + $arr["owner-name"] = $single_conv->actor->portablecontacts_net->displayName; $arr["owner-link"] = $actor; $arr["owner-avatar"] = $single_conv->actor->image->url; - //$arr["author-name"] = $single_conv->actor->contact->displayName; - //$arr["author-name"] = $single_conv->actor->contact->preferredUsername; $arr["author-name"] = $arr["owner-name"]; $arr["author-link"] = $actor; $arr["author-avatar"] = $single_conv->actor->image->url; @@ -284,33 +330,120 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio else $arr["app"] = "OStatus"; + $arr["app"] .= "*"; + + $arr["object"] = json_encode($single_conv); $arr["verb"] = $parent["verb"]; $arr["visible"] = $parent["visible"]; $arr["location"] = $single_conv->location->displayName; $arr["coord"] = trim($single_conv->location->lat." ".$single_conv->location->lon); + // Is it a reshared item? + if (isset($item->verb) AND ($item->verb == "share") AND isset($item->object)) { + if (is_array($item->object)) + $item->object = $item->object[0]; + + logger("Found reshared item ".$single_conv->object->id); + + // $single_conv->object->context->conversation; + + $plink = ostatus_convert_href($single_conv->object->url); + + $arr["uri"] = $single_conv->object->id; + $arr["plink"] = $plink; + $arr["created"] = $single_conv->object->published; + $arr["edited"] = $single_conv->object->published; + + $arr["author-name"] = $single_conv->object->actor->displayName; + if ($arr["owner-name"] == '') + $arr["author-name"] = $single_conv->object->actor->contact->displayName; + + $arr["author-link"] = $single_conv->object->actor->url; + $arr["author-avatar"] = $single_conv->object->actor->image->url; + + $arr["body"] = add_page_info_to_body(html2bbcode($single_conv->object->content)); + $arr["app"] = $single_conv->object->provider->displayName."#"; + //$arr["verb"] = $single_conv->object->verb; + + $arr["location"] = $single_conv->object->location->displayName; + $arr["coord"] = trim($single_conv->object->location->lat." ".$single_conv->object->location->lon); + } + if ($arr["location"] == "") unset($arr["location"]); if ($arr["coord"] == "") unset($arr["coord"]); + // Copy fields from given item array + if (isset($item["uri"]) AND ($item["uri"] == $arr["uri"])) { + $copy_fields = array("owner-name", "owner-link", "owner-avatar", "author-name", "author-link", "author-avatar", + "gravity", "body", "object-type", "verb", "created", "edited", "coord", "tag", + "attach", "app", "type", "location", "contact-id"); + foreach ($copy_fields AS $field) + if (isset($item[$field])) + $arr[$field] = $item[$field]; + + $arr["app"] .= "+"; + } + $newitem = item_store($arr); + if (!$newitem) { + logger("Item wasn't stored ".print_r($arr, true), LOGGER_DEBUG); + continue; + } - logger('Stored new item '.$plink.' for parent '.$arr["parent"].' under id '.$newitem, LOGGER_DEBUG); + if (isset($item["uri"]) AND ($item["uri"] == $arr["uri"])) { + $item = array(); + $item_stored = $newitem; + } + + logger('Stored new item '.$plink.' for parent '.$arr["parent-uri"].' under id '.$newitem, LOGGER_DEBUG); // Add the conversation entry (but don't fetch the whole conversation) complete_conversation($newitem, $conversation_url, true); // If the newly created item is the top item then change the parent settings of the thread // This shouldn't happen anymore. This is supposed to be absolote. - if ($newitem AND ($arr["uri"] == $first_id)) { + if ($arr["uri"] == $first_id) { logger('setting new parent to id '.$newitem); $new_parents = q("SELECT `id`, `uri`, `contact-id`, `type`, `verb`, `visible` FROM `item` WHERE `uid` = %d AND `id` = %d LIMIT 1", - intval($message["uid"]), intval($newitem)); + intval($uid), intval($newitem)); if ($new_parents) $parent = $new_parents[0]; } } +// Test +/* if ((count($item) > 0) AND ($item_stored <= 0)) { + $item_stored = item_store($item, true); + logger("In the conversation ".$conversation_url." the item uri ".$item["uri"]." wasn't found: ".$item_stored, LOGGER_DEBUG); + + if ($item_stored) + complete_conversation($item_id, $conversation_url, true); + } */ + + return($item_stored); +} + +function complete_conversation($itemid, $conversation_url, $only_add_conversation = false) { + global $a; + + $conversation_url = ostatus_convert_href($conversation_url); + + $messages = q("SELECT `uid`, `parent`, `created`, `received`, `guid` FROM `item` WHERE `id` = %d LIMIT 1", intval($itemid)); + if (!$messages) + return; + $message = $messages[0]; + + // Store conversation url if not done before + $conversation = q("SELECT `url` FROM `term` WHERE `uid` = %d AND `oid` = %d AND `otype` = %d AND `type` = %d", + intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION)); + + if (!$conversation) { + $r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`, `created`, `received`, `guid`) VALUES (%d, %d, %d, %d, '%s', '%s', '%s', '%s', '%s')", + intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION), + dbesc($message["created"]), dbesc($conversation_url), dbesc($message["created"]), dbesc($message["received"]), dbesc($message["guid"])); + logger('Storing conversation url '.$conversation_url.' for id '.$itemid); + } } ?> -- 2.39.5