]> git.mxchange.org Git - friendica.git/blobdiff - include/items.php
Merge branch 'master' of git://github.com/friendika/friendika
[friendica.git] / include / items.php
index 359378ff7d9296ead60a69268c33f6975b804402..dc177c468c9b21bc8e07ad3a08ea2363f2a70f64 100644 (file)
@@ -9,74 +9,27 @@ function get_feed_for(&$a, $dfrn_id, $owner_nick, $last_update, $direction = 0)
 
        // default permissions - anonymous user
 
-       $sql_extra = " 
-               AND `allow_cid` = '' 
-               AND `allow_gid` = '' 
-               AND `deny_cid`  = '' 
-               AND `deny_gid`  = '' 
-       ";
-
-       if(strlen($owner_nick) && ! intval($owner_nick)) {
-               $r = q("SELECT `uid`, `nickname`, `timezone` FROM `user` WHERE `nickname` = '%s' LIMIT 1",
-                       dbesc($owner_nick)
-               );
-               if(count($r)) {
-                       $owner_id = $r[0]['uid'];
-                       $owner_nick = $r[0]['nickname'];
-                       $owner_tz = $r[0]['timezone'];
-               }
-       }
-
-       $r = q("SELECT * FROM `contact` WHERE `self` = 1 AND `uid` = %d LIMIT 1",
-               intval($owner_id)
-       );
-       if(count($r)) {
-               $owner = $r[0];
-               $owner['nickname'] = $owner_nick;
-       }
-       else
+       if(! strlen($owner_nick))
                killme();
 
+       $sql_extra = " AND `allow_cid` = '' AND `allow_gid` = '' AND `deny_cid`  = '' AND `deny_gid`  = '' ";
 
-       /**
-        *
-        * Determine the next birthday, but only if the birthday is published
-        * in the default profile. We _could_ also look for a private profile that the
-        * recipient can see, but somebody could get mad at us if they start getting
-        * public birthday greetings when they haven't made this info public. 
-        *
-        * Assuming we are able to publish this info, we are then going to convert
-        * the start time from the owner's timezone to UTC. 
-        *
-        * This will potentially solve the problem found with some social networks
-        * where birthdays are converted to the viewer's timezone and salutations from
-        * elsewhere in the world show up on the wrong day. We will convert it to the
-        * viewer's timezone also, but first we are going to convert it from the birthday
-        * person's timezone to GMT - so the viewer may find the birthday starting at
-        * 6:00PM the day before, but that will correspond to midnight to the birthday person.
-        *
-        */
+       $r = q("SELECT `contact`.*, `user`.`uid` AS `user_uid`, `user`.`nickname`, `user`.`timezone`
+               FROM `contact` LEFT JOIN `user` ON `user`.`uid` = `contact`.`uid`
+               WHERE `contact`.`self` = 1 AND `user`.`nickname` = '%s' LIMIT 1",
+               dbesc($owner_nick)
+       );
 
-       $birthday = '';
+       if(! count($r))
+               killme();
 
-       $p = q("SELECT `dob` FROM `profile` WHERE `is-default` = 1 AND `uid` = %d LIMIT 1",
-               intval($owner_id)
-       );
+       $owner = $r[0];
+       $owner_id = $owner['user_uid'];
+       $owner_nick = $owner['nickname'];
 
-       if($p && count($p)) {
-               $tmp_dob = substr($p[0]['dob'],5);
-               if(intval($tmp_dob)) {
-                       $y = datetime_convert($owner_tz,$owner_tz,'now','Y');
-                       $bd = $y . '-' . $tmp_dob . ' 00:00';
-                       $t_dob = strtotime($bd);
-                       $now = strtotime(datetime_convert($owner_tz,$owner_tz,'now'));
-                       if($t_dob < $now)
-                               $bd = $y + 1 . '-' . $tmp_dob . ' 00:00';
-                       $birthday = datetime_convert($owner_tz,'UTC',$bd,ATOM_TIME); 
-               }
-       }
+       $birthday = feed_birthday($owner_id,$owner['timezone']);
 
-       if($dfrn_id && $dfrn_id != '*') {
+       if(strlen($dfrn_id)) {
 
                $sql_extra = '';
                switch($direction) {
@@ -102,7 +55,7 @@ function get_feed_for(&$a, $dfrn_id, $owner_nick, $last_update, $direction = 0)
                );
 
                if(! count($r))
-                       return false;
+                       killme();
 
                $contact = $r[0];
                $groups = init_groups_visitor($contact['id']);
@@ -144,7 +97,7 @@ function get_feed_for(&$a, $dfrn_id, $owner_nick, $last_update, $direction = 0)
                `contact`.`thumb`, `contact`.`dfrn-id`, `contact`.`self`, 
                `contact`.`id` AS `contact-id`, `contact`.`uid` AS `contact-uid`
                FROM `item` LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
-               WHERE `item`.`uid` = %d AND `item`.`visible` = 1 
+               WHERE `item`.`uid` = %d AND `item`.`visible` = 1 AND `item`.`parent` != 0 
                AND `item`.`wall` = 1 AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0
                AND ( `item`.`edited` > '%s' OR `item`.`changed` > '%s' )
                $sql_extra
@@ -156,33 +109,17 @@ function get_feed_for(&$a, $dfrn_id, $owner_nick, $last_update, $direction = 0)
        );
 
        // Will check further below if this actually returned results.
-       // We will provide an empty feed in any case.
+       // We will provide an empty feed if that is the case.
 
        $items = $r;
 
-       $feed_template = load_view_file('view/atom_feed.tpl');
+       $feed_template = get_markup_template('atom_feed.tpl');
 
        $atom = '';
 
-       $hub = get_config('system','huburl');
-
-       $hubxml = '';
-       if(strlen($hub)) {
-               $hubs = explode(',', $hub);
-               if(count($hubs)) {
-                       foreach($hubs as $h) {
-                               $h = trim($h);
-                               if(! strlen($h))
-                                       continue;
-                               $hubxml .= '<link rel="hub" href="' . xmlify($h) . '" />' . "\n" ;
-                       }
-               }
-       }
-
-       $salmon = '<link rel="salmon" href="' . xmlify($a->get_baseurl() . '/salmon/' . $owner_nick) . '" />' . "\n" ; 
-       $salmon .= '<link rel="http://salmon-protocol.org/ns/salmon-replies" href="' . xmlify($a->get_baseurl() . '/salmon/' . $owner_nick) . '" />' . "\n" ; 
-       $salmon .= '<link rel="http://salmon-protocol.org/ns/salmon-mention" href="' . xmlify($a->get_baseurl() . '/salmon/' . $owner_nick) . '" />' . "\n" ; 
+       $hubxml = feed_hublinks();
 
+       $salmon = feed_salmonlinks($owner_nick);
 
        $atom .= replace_macros($feed_template, array(
                '$version'      => xmlify(FRIENDIKA_VERSION),
@@ -215,7 +152,7 @@ function get_feed_for(&$a, $dfrn_id, $owner_nick, $last_update, $direction = 0)
 
                // public feeds get html, our own nodes use bbcode
 
-               if($dfrn_id === '*') {
+               if($dfrn_id === '') {
                        $type = 'html';
                }
                else {
@@ -243,7 +180,11 @@ function construct_activity_object($item) {
 
        if($item['object']) {
                $o = '<as:object>' . "\r\n";
-               $r = @simplexml_load_string($item['object']);
+               $r = parse_xml_string($item['object'],false);
+
+
+               if(! $r)
+                       return '';
                if($r->type)
                        $o .= '<as:object-type>' . xmlify($r->type) . '</as:object-type>' . "\r\n";
                if($r->id)
@@ -251,8 +192,15 @@ function construct_activity_object($item) {
                if($r->title)
                        $o .= '<title>' . xmlify($r->title) . '</title>' . "\r\n";
                if($r->link) {
-                       if(substr($r->link,0,1) === '<') 
+                       if(substr($r->link,0,1) === '<') {
+                               // patch up some facebook "like" activity objects that got stored incorrectly
+                               // for a couple of months prior to 9-Jun-2011 and generated bad XML.
+                               // we can probably remove this hack here and in the following function in a few months time.
+                               if(strstr($r->link,'&') && (! strstr($r->link,'&amp;')))
+                                       $r->link = str_replace('&','&amp;', $r->link);
+                               $r->link = preg_replace('/\<link(.*?)\"\>/','<link$1"/>',$r->link);
                                $o .= $r->link;
+                       }                                       
                        else
                                $o .= '<link rel="alternate" type="text/html" href="' . xmlify($r->link) . '" />' . "\r\n";
                }
@@ -269,7 +217,9 @@ function construct_activity_target($item) {
 
        if($item['target']) {
                $o = '<as:target>' . "\r\n";
-               $r = @simplexml_load_string($item['target']);
+               $r = parse_xml_string($item['target'],false);
+               if(! $r)
+                       return '';
                if($r->type)
                        $o .= '<as:object-type>' . xmlify($r->type) . '</as:object-type>' . "\r\n";
                if($r->id)
@@ -277,8 +227,12 @@ function construct_activity_target($item) {
                if($r->title)
                        $o .= '<title>' . xmlify($r->title) . '</title>' . "\r\n";
                if($r->link) {
-                       if(substr($r->link,0,1) === '<') 
+                       if(substr($r->link,0,1) === '<') {
+                               if(strstr($r->link,'&') && (! strstr($r->link,'&amp;')))
+                                       $r->link = str_replace('&','&amp;', $r->link);
+                               $r->link = preg_replace('/\<link(.*?)\"\>/','<link$1"/>',$r->link);
                                $o .= $r->link;
+                       }                                       
                        else
                                $o .= '<link rel="alternate" type="text/html" href="' . xmlify($r->link) . '" />' . "\r\n";
                }
@@ -304,12 +258,18 @@ function get_atom_elements($feed,$item) {
        $res = array();
 
        $author = $item->get_author();
-       $res['author-name'] = unxmlify($author->get_name());
-       $res['author-link'] = unxmlify($author->get_link());
+       if($author) { 
+               $res['author-name'] = unxmlify($author->get_name());
+               $res['author-link'] = unxmlify($author->get_link());
+       }
+       else {
+               $res['author-name'] = unxmlify($feed->get_title());
+               $res['author-link'] = unxmlify($feed->get_permalink());
+       }
        $res['uri'] = unxmlify($item->get_id());
        $res['title'] = unxmlify($item->get_title());
        $res['body'] = unxmlify($item->get_content());
-
+       $res['plink'] = unxmlify($item->get_link(0));
 
        // look for a photo. We should check media size and find the best one,
        // but for now let's just find any author photo
@@ -406,14 +366,16 @@ function get_atom_elements($feed,$item) {
        // the wild, by sanitising it and converting supported tags to bbcode before we rip out any remaining 
        // html.
 
-
        if((strpos($res['body'],'<') !== false) || (strpos($res['body'],'>') !== false)) {
 
                $res['body'] = preg_replace('#<object[^>]+>.+?' . 'http://www.youtube.com/((?:v|cp)/[A-Za-z0-9\-_=]+).+?</object>#s',
                        '[youtube]$1[/youtube]', $res['body']);
 
+               $res['body'] = preg_replace('#<iframe[^>].+?' . 'http://www.youtube.com/embed/([A-Za-z0-9\-_=]+).+?</iframe>#s',
+                       '[youtube]$1[/youtube]', $res['body']);
+
                $res['body'] = oembed_html2bbcode($res['body']);
-       
+
                $config = HTMLPurifier_Config::createDefault();
                $config->set('Cache.DefinitionImpl', null);
 
@@ -426,7 +388,7 @@ function get_atom_elements($feed,$item) {
 
                $res['body'] = html2bbcode($res['body']);
        }
-       
+
        $allow = $item->get_item_tags(NAMESPACE_DFRN,'comment-allow');
        if($allow && $allow[0]['data'] == 1)
                $res['last-child'] = 1;
@@ -439,6 +401,9 @@ function get_atom_elements($feed,$item) {
        else
                $res['private'] = 0;
 
+       $extid = $item->get_item_tags(NAMESPACE_DFRN,'extid');
+       if($extid && $extid[0]['data'])
+               $res['extid'] = $extid[0]['data'];
 
        $rawlocation = $item->get_item_tags(NAMESPACE_DFRN, 'location');
        if($rawlocation)
@@ -464,6 +429,17 @@ function get_atom_elements($feed,$item) {
                $res['edited'] = $item->get_date('c');
 
 
+       // Disallow time travelling posts
+
+       $d1 = strtotime($res['created']);
+       $d2 = strtotime($res['edited']);
+       $d3 = strtotime('now');
+
+       if($d1 > $d3)
+               $res['created'] = datetime_convert();
+       if($d2 > $d3)
+               $res['edited'] = datetime_convert();
+
        $rawowner = $item->get_item_tags(NAMESPACE_DFRN, 'owner');
        if($rawowner[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['name'][0]['data'])
                $res['owner-name'] = unxmlify($rawowner[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['name'][0]['data']);
@@ -503,7 +479,45 @@ function get_atom_elements($feed,$item) {
        if((x($res,'verb')) && ($res['verb'] === 'http://ostatus.org/schema/1.0/unfollow'))
                $res['verb'] = ACTIVITY_UNFOLLOW;
 
-               
+
+       $cats = $item->get_categories();
+       if($cats) {
+               $tag_arr = array();
+               foreach($cats as $cat) {
+                       $term = $cat->get_term();
+                       if(! $term)
+                               $term = $cat->get_label();
+                       $scheme = $cat->get_scheme();
+                       if($scheme && $term && stristr($scheme,'X-DFRN:'))
+                               $tag_arr[] = substr($scheme,7,1) . '[url=' . unxmlify(substr($scheme,9)) . ']' . unxmlify($term) . '[/url]';
+                       elseif($term)
+                               $tag_arr[] = notags(trim($term));
+               }
+               $res['tag'] =  implode(',', $tag_arr);
+       }
+
+       $attach = $item->get_enclosures();
+       if($attach) {
+               $att_arr = array();
+               foreach($attach as $att) {
+                       $len   = intval($att->get_length());
+                       $link  = str_replace(array(',','"'),array('%2D','%22'),notags(trim(unxmlify($att->get_link()))));
+                       $title = str_replace(array(',','"'),array('%2D','%22'),notags(trim(unxmlify($att->get_title()))));
+                       $type  = str_replace(array(',','"'),array('%2D','%22'),notags(trim(unxmlify($att->get_type()))));
+                       if(strpos($type,';'))
+                               $type = substr($type,0,strpos($type,';'));
+                       if((! $link) || (strpos($link,'http') !== 0))
+                               continue;
+
+                       if(! $title)
+                               $title = ' ';
+                       if(! $type)
+                               $type = 'application/octet-stream';
+
+                       $att_arr[] = '[attach]href="' . $link . '" size="' . $len . '" type="' . $type . '" title="' . $title . '"[/attach]'; 
+               }
+               $res['attach'] = implode(',', $att_arr);
+       }
 
        $rawobj = $item->get_item_tags(NAMESPACE_ACTIVITY, 'object');
 
@@ -530,6 +544,10 @@ function get_atom_elements($feed,$item) {
                                $body = preg_replace('#<object[^>]+>.+?' . 'http://www.youtube.com/((?:v|cp)/[A-Za-z0-9\-_=]+).+?</object>#s',
                                        '[youtube]$1[/youtube]', $body);
 
+               $res['body'] = preg_replace('#<iframe[^>].+?' . 'http://www.youtube.com/embed/([A-Za-z0-9\-_=]+).+?</iframe>#s',
+                       '[youtube]$1[/youtube]', $res['body']);
+
+
                                $config = HTMLPurifier_Config::createDefault();
                                $config->set('Cache.DefinitionImpl', null);
 
@@ -569,6 +587,9 @@ function get_atom_elements($feed,$item) {
                                $body = preg_replace('#<object[^>]+>.+?' . 'http://www.youtube.com/((?:v|cp)/[A-Za-z0-9\-_=]+).+?</object>#s',
                                        '[youtube]$1[/youtube]', $body);
 
+               $res['body'] = preg_replace('#<iframe[^>].+?' . 'http://www.youtube.com/embed/([A-Za-z0-9\-_=]+).+?</iframe>#s',
+                       '[youtube]$1[/youtube]', $res['body']);
+
                                $config = HTMLPurifier_Config::createDefault();
                                $config->set('Cache.DefinitionImpl', null);
 
@@ -633,6 +654,7 @@ function item_store($arr,$force_parent = false) {
 
        $arr['wall']          = ((x($arr,'wall'))          ? intval($arr['wall'])                : 0);
        $arr['uri']           = ((x($arr,'uri'))           ? notags(trim($arr['uri']))           : random_string());
+       $arr['extid']         = ((x($arr,'extid'))         ? notags(trim($arr['extid']))         : '');
        $arr['author-name']   = ((x($arr,'author-name'))   ? notags(trim($arr['author-name']))   : '');
        $arr['author-link']   = ((x($arr,'author-link'))   ? notags(trim($arr['author-link']))   : '');
        $arr['author-avatar'] = ((x($arr,'author-avatar')) ? notags(trim($arr['author-avatar'])) : '');
@@ -654,12 +676,15 @@ function item_store($arr,$force_parent = false) {
        $arr['object']        = ((x($arr,'object'))        ? trim($arr['object'])                : '');
        $arr['target-type']   = ((x($arr,'target-type'))   ? notags(trim($arr['target-type']))   : '');
        $arr['target']        = ((x($arr,'target'))        ? trim($arr['target'])                : '');
+       $arr['plink']         = ((x($arr,'plink'))         ? notags(trim($arr['plink']))         : '');
        $arr['allow_cid']     = ((x($arr,'allow_cid'))     ? trim($arr['allow_cid'])             : '');
        $arr['allow_gid']     = ((x($arr,'allow_gid'))     ? trim($arr['allow_gid'])             : '');
        $arr['deny_cid']      = ((x($arr,'deny_cid'))      ? trim($arr['deny_cid'])              : '');
        $arr['deny_gid']      = ((x($arr,'deny_gid'))      ? trim($arr['deny_gid'])              : '');
        $arr['private']       = ((x($arr,'private'))       ? intval($arr['private'])             : 0 );
        $arr['body']          = ((x($arr,'body'))          ? trim($arr['body'])                  : '');
+       $arr['tag']           = ((x($arr,'tag'))           ? notags(trim($arr['tag']))           : '');
+       $arr['attach']        = ((x($arr,'attach'))        ? notags(trim($arr['attach']))        : '');
 
        if($arr['parent-uri'] === $arr['uri']) {
                $parent_id = 0;
@@ -671,6 +696,7 @@ function item_store($arr,$force_parent = false) {
        else { 
 
                // find the parent and snarf the item id and ACL's
+               // and anything else we need to inherit
 
                $r = q("SELECT * FROM `item` WHERE `uri` = '%s' AND `uid` = %d LIMIT 1",
                        dbesc($arr['parent-uri']),
@@ -694,6 +720,7 @@ function item_store($arr,$force_parent = false) {
                        $allow_gid      = $r[0]['allow_gid'];
                        $deny_cid       = $r[0]['deny_cid'];
                        $deny_gid       = $r[0]['deny_gid'];
+                       $arr['wall']    = $r[0]['wall'];
                }
                else {
 
@@ -705,6 +732,7 @@ function item_store($arr,$force_parent = false) {
                                $parent_id = 0;
                                $arr['thr-parent'] = $arr['parent-uri'];
                                $arr['parent-uri'] = $arr['uri'];
+                               $arr['gravity'] = 0;
                        }
                        else {
                                logger('item_store: item parent was not found - ignoring item');
@@ -731,6 +759,15 @@ function item_store($arr,$force_parent = false) {
                $arr['uri'],           // already dbesc'd
                intval($arr['uid'])
        );
+       if(! count($r)) {
+               // This is not good, but perhaps we encountered a rare race/cache condition, so back off and try again. 
+               sleep(3);
+               $r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' AND `uid` = %d LIMIT 1",
+                       $arr['uri'],           // already dbesc'd
+                       intval($arr['uid'])
+               );
+       }
+
        if(count($r)) {
                $current_post = $r[0]['id'];
                logger('item_store: created item ' . $current_post);
@@ -762,6 +799,18 @@ function item_store($arr,$force_parent = false) {
                intval($current_post)
        );
 
+       /**
+        * If this is now the last-child, force all _other_ children of this parent to *not* be last-child
+        */
+
+       if($arr['last-child']) {
+               $r = q("UPDATE `item` SET `last-child` = 0 WHERE `parent-uri` = '%s' AND `uid` = %d AND `id` != %d",
+                       dbesc($arr['uri']),
+                       intval($arr['uid']),
+                       intval($current_post)
+               );
+       }
+
        return $current_post;
 }
 
@@ -799,7 +848,7 @@ function dfrn_deliver($owner,$contact,$atom, $dissolve = false) {
        if(! $rino_enable)
                $rino = 0;
 
-       $url = $contact['notify'] . '?dfrn_id=' . $idtosend . '&dfrn_version=' . DFRN_PROTOCOL_VERSION . (($rino) ? '&rino=1' : '');
+       $url = $contact['notify'] . '&dfrn_id=' . $idtosend . '&dfrn_version=' . DFRN_PROTOCOL_VERSION . (($rino) ? '&rino=1' : '');
 
        logger('dfrn_deliver: ' . $url);
 
@@ -820,7 +869,7 @@ function dfrn_deliver($owner,$contact,$atom, $dissolve = false) {
                return 3;
        }
 
-       $res = simplexml_load_string($xml);
+       $res = parse_xml_string($xml);
 
        if((intval($res->status) != 0) || (! strlen($res->challenge)) || (! strlen($res->dfrn_id)))
                return (($res->status) ? $res->status : 3);
@@ -859,14 +908,14 @@ function dfrn_deliver($owner,$contact,$atom, $dissolve = false) {
        if($dissolve)
                $postvars['dissolve'] = '1';
 
-       if(($contact['rel']) && ($contact['rel'] != REL_FAN) && (! $contact['blocked']) && (! $contact['readonly'])) {
-               $postvars['data'] = $atom;
-       }
-       elseif($owner['page-flags'] == PAGE_COMMUNITY) {
+
+       if((($contact['rel']) && ($contact['rel'] != REL_FAN) && (! $contact['blocked'])) || ($owner['page-flags'] == PAGE_COMMUNITY)) {
                $postvars['data'] = $atom;
+               $postvars['perm'] = 'rw';
        }
        else {
                $postvars['data'] = str_replace('<dfrn:comment-allow>1','<dfrn:comment-allow>0',$atom);
+               $postvars['perm'] = 'r';
        }
 
        if($rino && $rino_allowed && (! $dissolve)) {
@@ -908,26 +957,24 @@ function dfrn_deliver($owner,$contact,$atom, $dissolve = false) {
        if((! $curl_stat) || (! strlen($xml)))
                return(-1); // timed out
 
-
        if(strpos($xml,'<?xml') === false) {
                logger('dfrn_deliver: phase 2: no valid XML returned');
                logger('dfrn_deliver: phase 2: returned XML: ' . $xml, LOGGER_DATA);
                return 3;
        }
 
-       $res = simplexml_load_string($xml);
+       $res = parse_xml_string($xml);
 
-       return $res->status;
+       return $res->status; 
 }
 
 
-/*
+/**
  *
  * consume_feed - process atom feed and update anything/everything we might need to update
  *
- * $xml = the (atom) feed to consume - no RSS spoken here, it might partially work since simplepie 
- *        handles both, but we don't claim it will work well, and are reasonably certain it won't.
+ * $xml = the (atom) feed to consume - RSS isn't as fully supported but may work for simple feeds.
+ *
  * $importer = the contact_record (joined to user_record) of the local user who owns this relationship.
  *             It is this person's stuff that is going to be updated.
  * $contact =  the person who is sending us stuff. If not set, we MAY be processing a "follow" activity
@@ -938,7 +985,7 @@ function dfrn_deliver($owner,$contact,$atom, $dissolve = false) {
  *
  */
 
-function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0) {
+function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $secure_feed = false) {
 
        require_once('simplepie/simplepie.inc');
 
@@ -953,6 +1000,7 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0) {
        if($feed->error())
                logger('consume_feed: Error parsing XML: ' . $feed->error());
 
+       $permalink = $feed->get_permalink();
 
        // Check at the feed level for updated contact name and/or photo
 
@@ -1094,25 +1142,18 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0) {
 
        }
 
-       // Now process the feed
-       if($feed->get_item_quantity()) {                
-
-        // in inverse date order
-               if ($datedir)
-                       $items = array_reverse($feed->get_items());
-               else
-                       $items = $feed->get_items();
 
-               foreach($items as $item) {
+       // process any deleted entries
 
+       $del_entries = $feed->get_feed_tags(NAMESPACE_TOMB, 'deleted-entry');
+       if(is_array($del_entries) && count($del_entries)) {
+               foreach($del_entries as $dentry) {
                        $deleted = false;
-
-                       $rawdelete = $item->get_item_tags( NAMESPACE_TOMB, 'deleted-entry');
-                       if(isset($rawdelete[0]['attribs']['']['ref'])) {
-                               $uri = $rawthread[0]['attribs']['']['ref'];
+                       if(isset($dentry['attribs']['']['ref'])) {
+                               $uri = $dentry['attribs']['']['ref'];
                                $deleted = true;
-                               if(isset($rawdelete[0]['attribs']['']['when'])) {
-                                       $when = $rawthread[0]['attribs']['']['when'];
+                               if(isset($dentry['attribs']['']['when'])) {
+                                       $when = $dentry['attribs']['']['when'];
                                        $when = datetime_convert('UTC','UTC', $when, 'Y-m-d H:i:s');
                                }
                                else
@@ -1126,6 +1167,10 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0) {
                                );
                                if(count($r)) {
                                        $item = $r[0];
+
+                                       if(! $item['deleted'])
+                                               logger('consume_feed: deleting item ' . $item['id'] . ' uri=' . $item['uri'], LOGGER_DEBUG);
+
                                        if($item['uri'] == $item['parent-uri']) {
                                                $r = q("UPDATE `item` SET `deleted` = 1, `edited` = '%s', `changed` = '%s',
                                                        `body` = '', `title` = ''
@@ -1147,7 +1192,7 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0) {
                                                );
                                                if($item['last-child']) {
                                                        // ensure that last-child is set in case the comment that had it just got wiped.
-                                                       $q("UPDATE `item` SET `last-child` = 0, `changed` = '%s' WHERE `parent-uri` = '%s' AND `uid` = %d ",
+                                                       q("UPDATE `item` SET `last-child` = 0, `changed` = '%s' WHERE `parent-uri` = '%s' AND `uid` = %d ",
                                                                dbesc(datetime_convert()),
                                                                dbesc($item['parent-uri']),
                                                                intval($item['uid'])
@@ -1166,9 +1211,24 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0) {
                                                }       
                                        }
                                }       
-                               continue;
                        }
+               }
+       }
 
+       // Now process the feed
+
+       if($feed->get_item_quantity()) {                
+
+               logger('consume_feed: feed item count = ' . $feed->get_item_quantity());
+
+        // in inverse date order
+               if ($datedir)
+                       $items = array_reverse($feed->get_items());
+               else
+                       $items = $feed->get_items();
+
+
+               foreach($items as $item) {
 
                        $is_reply = false;              
                        $item_id = $item->get_id();
@@ -1178,19 +1238,39 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0) {
                                $parent_uri = $rawthread[0]['attribs']['']['ref'];
                        }
 
-
                        if(($is_reply) && is_array($contact)) {
-       
+
                                // Have we seen it? If not, import it.
        
-                               $item_id = $item->get_id();
-       
-                               $r = q("SELECT `uid`, `last-child`, `edited` FROM `item` WHERE `uri` = '%s' AND `uid` = %d LIMIT 1",
+                               $item_id  = $item->get_id();
+                               $datarray = get_atom_elements($feed,$item);
+
+                               if(! x($datarray,'author-name'))
+                                       $datarray['author-name'] = $contact['name'];
+                               if(! x($datarray,'author-link'))
+                                       $datarray['author-link'] = $contact['url'];
+                               if(! x($datarray,'author-avatar'))
+                                       $datarray['author-avatar'] = $contact['thumb'];
+
+                               $r = q("SELECT `uid`, `last-child`, `edited`, `body` FROM `item` WHERE `uri` = '%s' AND `uid` = %d LIMIT 1",
                                        dbesc($item_id),
                                        intval($importer['uid'])
                                );
-                               // FIXME update content if 'updated' changes
+
+                               // Update content if 'updated' changes
+
                                if(count($r)) {
+                                       if((x($datarray,'edited') !== false) && (datetime_convert('UTC','UTC',$datarray['edited']) !== $r[0]['edited'])) {  
+                                               $r = q("UPDATE `item` SET `body` = '%s', `edited` = '%s' WHERE `uri` = '%s' AND `uid` = %d LIMIT 1",
+                                                       dbesc($datarray['body']),
+                                                       dbesc(datetime_convert('UTC','UTC',$datarray['edited'])),
+                                                       dbesc($item_id),
+                                                       intval($importer['uid'])
+                                               );
+                                       }
+
+                                       // update last-child if it changes
+
                                        $allow = $item->get_item_tags( NAMESPACE_DFRN, 'comment-allow');
                                        if(($allow) && ($allow[0]['data'] != $r[0]['last-child'])) {
                                                $r = q("UPDATE `item` SET `last-child` = 0, `changed` = '%s' WHERE `parent-uri` = '%s' AND `uid` = %d",
@@ -1207,7 +1287,7 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0) {
                                        }
                                        continue;
                                }
-                               $datarray = get_atom_elements($feed,$item);
+
                                $force_parent = false;
                                if($contact['network'] === 'stat') {
                                        $force_parent = true;
@@ -1238,14 +1318,60 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0) {
                        }
 
                        else {
+
                                // Head post of a conversation. Have we seen it? If not, import it.
 
-                               $item_id = $item->get_id();
-                               $r = q("SELECT `uid`, `last-child`, `edited` FROM `item` WHERE `uri` = '%s' AND `uid` = %d LIMIT 1",
+                               $item_id  = $item->get_id();
+
+                               $datarray = get_atom_elements($feed,$item);
+
+                               if(is_array($contact)) {
+                                       if(! x($datarray,'author-name'))
+                                               $datarray['author-name'] = $contact['name'];
+                                       if(! x($datarray,'author-link'))
+                                               $datarray['author-link'] = $contact['url'];
+                                       if(! x($datarray,'author-avatar'))
+                                               $datarray['author-avatar'] = $contact['thumb'];
+                               }
+
+                               if((x($datarray,'object-type')) && ($datarray['object-type'] === ACTIVITY_OBJ_EVENT)) {
+                                       $ev = bbtoevent($datarray['body']);
+                                       if(x($ev,'desc') && x($ev,'start')) {
+                                               $ev['uid'] = $importer['uid'];
+                                               $ev['uri'] = $item_id;
+
+                                               if(is_array($contact))
+                                                       $ev['cid'] = $contact['id'];
+                                               $r = q("SELECT * FROM `event` WHERE `uri` = '%s' AND `uid` = %d LIMIT 1",
+                                                       dbesc($item_id),
+                                                       intval($importer['uid'])
+                                               );
+                                               if(count($r))
+                                                       $ev['id'] = $r[0]['id'];
+                                               $xyz = event_store($ev);
+                                               continue;
+                                       }
+                               }
+
+                               $r = q("SELECT `uid`, `last-child`, `edited`, `body` FROM `item` WHERE `uri` = '%s' AND `uid` = %d LIMIT 1",
                                        dbesc($item_id),
                                        intval($importer['uid'])
                                );
+
+                               // Update content if 'updated' changes
+
                                if(count($r)) {
+                                       if((x($datarray,'edited') !== false) && (datetime_convert('UTC','UTC',$datarray['edited']) !== $r[0]['edited'])) {  
+                                               $r = q("UPDATE `item` SET `body` = '%s', `edited` = '%s' WHERE `uri` = '%s' AND `uid` = %d LIMIT 1",
+                                                       dbesc($datarray['body']),
+                                                       dbesc(datetime_convert('UTC','UTC',$datarray['edited'])),
+                                                       dbesc($item_id),
+                                                       intval($importer['uid'])
+                                               );
+                                       }
+
+                                       // update last-child if it changes
+
                                        $allow = $item->get_item_tags( NAMESPACE_DFRN, 'comment-allow');
                                        if($allow && $allow[0]['data'] != $r[0]['last-child']) {
                                                $r = q("UPDATE `item` SET `last-child` = %d , `changed` = '%s' WHERE `uri` = '%s' AND `uid` = %d LIMIT 1",
@@ -1257,7 +1383,6 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0) {
                                        }
                                        continue;
                                }
-                               $datarray = get_atom_elements($feed,$item);
 
                                if(activity_match($datarray['verb'],ACTIVITY_FOLLOW)) {
                                        logger('consume-feed: New follower');
@@ -1271,7 +1396,7 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0) {
                                if(! is_array($contact))
                                        return;
 
-                               if($contact['network'] === 'stat') {
+                               if($contact['network'] === 'stat' || stristr($permalink,'twitter.com')) {
                                        if(strlen($datarray['title']))
                                                unset($datarray['title']);
                                        $datarray['last-child'] = 1;
@@ -1318,8 +1443,8 @@ function new_follower($importer,$contact,$datarray,$item) {
                // create contact record - set to readonly
 
                $r = q("INSERT INTO `contact` ( `uid`, `created`, `url`, `name`, `nick`, `photo`, `network`, `rel`, 
-                       `blocked`, `readonly`, `pending` )
-                       VALUES ( %d, '%s', '%s', '%s', '%s', '%s', '%s', %d, 0, 1, 1 ) ",
+                       `blocked`, `readonly`, `pending`, `writable` )
+                       VALUES ( %d, '%s', '%s', '%s', '%s', '%s', '%s', %d, 0, 1, 1, 1 ) ",
                        intval($importer['uid']),
                        dbesc(datetime_convert()),
                        dbesc($url),
@@ -1355,7 +1480,7 @@ function new_follower($importer,$contact,$datarray,$item) {
                $a = get_app();
                if(count($r)) {
                        if(($r[0]['notify-flags'] & NOTIFY_INTRO) && ($r[0]['page-flags'] == PAGE_NORMAL)) {
-                               $email_tpl = load_view_file('view/follow_notify_eml.tpl');
+                               $email_tpl = get_intltext_template('follow_notify_eml.tpl');
                                $email = replace_macros($email_tpl, array(
                                        '$requestor' => ((strlen($name)) ? $name : t('[Name Withheld]')),
                                        '$url' => $url,
@@ -1366,7 +1491,9 @@ function new_follower($importer,$contact,$datarray,$item) {
                                $res = mail($r[0]['email'], 
                                        t("You have a new follower at ") . $a->config['sitename'],
                                        $email,
-                                       'From: ' . t('Administrator') . '@' . $_SERVER['SERVER_NAME'] );
+                                       'From: ' . t('Administrator') . '@' . $_SERVER['SERVER_NAME'] . "\n"
+                                       . 'Content-type: text/plain; charset=UTF-8' . "\n"
+                                       . 'Content-transfer-encoding: 8bit' );
                        
                        }
                }
@@ -1445,17 +1572,24 @@ function atom_author($tag,$name,$uri,$h,$w,$photo) {
 
 function atom_entry($item,$type,$author,$owner,$comment = false) {
 
+       $a = get_app();
+
        if($item['deleted'])
                return '<at:deleted-entry ref="' . xmlify($item['uri']) . '" when="' . xmlify(datetime_convert('UTC','UTC',$item['edited'] . '+00:00',ATOM_TIME)) . '" />' . "\r\n";
 
-       $a = get_app();
+
+       if($item['allow_cid'] || $item['allow_gid'] || $item['deny_cid'] || $item['deny_gid'])
+               $body = fix_private_photos($item['body'],$owner['uid']);
+       else
+               $body = $item['body'];
+
 
        $o = "\r\n\r\n<entry>\r\n";
 
        if(is_array($author))
                $o .= atom_author('author',$author['name'],$author['url'],80,80,$author['thumb']);
        else
-               $o .= atom_author('author',$item['name'],$item['url'],80,80,$item['thumb']);
+               $o .= atom_author('author',(($item['author-name']) ? $item['author-name'] : $item['name']),(($item['author-link']) ? $item['author-link'] : $item['url']),80,80,(($item['author-avatar']) ? $item['author-avatar'] : $item['thumb']));
        if(strlen($item['owner-name']))
                $o .= atom_author('dfrn:owner',$item['owner-name'],$item['owner-link'],80,80,$item['owner-avatar']);
 
@@ -1466,9 +1600,9 @@ function atom_entry($item,$type,$author,$owner,$comment = false) {
        $o .= '<title>' . xmlify($item['title']) . '</title>' . "\r\n";
        $o .= '<published>' . xmlify(datetime_convert('UTC','UTC',$item['created'] . '+00:00',ATOM_TIME)) . '</published>' . "\r\n";
        $o .= '<updated>' . xmlify(datetime_convert('UTC','UTC',$item['edited'] . '+00:00',ATOM_TIME)) . '</updated>' . "\r\n";
-       $o .= '<dfrn:env>' . base64url_encode($item['body'], true) . '</dfrn:env>' . "\r\n";
-       $o .= '<content type="' . $type . '" >' . xmlify(($type === 'html') ? bbcode($item['body']) : $item['body']) . '</content>' . "\r\n";
-       $o .= '<link rel="alternate" href="' . xmlify($a->get_baseurl() . '/display/' . $owner['nickname'] . '/' . $item['id']) . '" />' . "\r\n";
+       $o .= '<dfrn:env>' . base64url_encode($body, true) . '</dfrn:env>' . "\r\n";
+       $o .= '<content type="' . $type . '" >' . xmlify(($type === 'html') ? bbcode($body) : $body) . '</content>' . "\r\n";
+       $o .= '<link rel="alternate" type="text/html" href="' . xmlify($a->get_baseurl() . '/display/' . $owner['nickname'] . '/' . $item['id']) . '" />' . "\r\n";
        if($comment)
                $o .= '<dfrn:comment-allow>' . intval($item['last-child']) . '</dfrn:comment-allow>' . "\r\n";
 
@@ -1483,6 +1617,10 @@ function atom_entry($item,$type,$author,$owner,$comment = false) {
        if(($item['private']) || strlen($item['allow_cid']) || strlen($item['allow_gid']) || strlen($item['deny_cid']) || strlen($item['deny_gid']))
                $o .= '<dfrn:private>1</dfrn:private>' . "\r\n";
 
+       if($item['extid'])
+               $o .= '<dfrn:extid>' . $item['extid'] . '</dfrn:extid>' . "\r\n";
+
+
        $verb = construct_verb($item);
        $o .= '<as:verb>' . xmlify($verb) . '</as:verb>' . "\r\n";
        $actobj = construct_activity_object($item);
@@ -1492,6 +1630,15 @@ function atom_entry($item,$type,$author,$owner,$comment = false) {
        if(strlen($actarg))
                $o .= $actarg;
 
+       $tags = item_getfeedtags($item);
+       if(count($tags)) {
+               foreach($tags as $t) {
+                       $o .= '<category scheme="X-DFRN:' . xmlify($t[0]) . ':' . xmlify($t[1]) . '" term="' . xmlify($t[2]) . '" />' . "\r\n";
+               }
+       }
+
+       $o .= item_getfeedattach($item);
+
        $mentioned = get_mentions($item);
        if($mentioned)
                $o .= $mentioned;
@@ -1502,4 +1649,126 @@ function atom_entry($item,$type,$author,$owner,$comment = false) {
        
        return $o;
 }
+
+function fix_private_photos($s,$uid) {
+       $a = get_app();
+       logger('fix_private_photos');
+
+       if(preg_match("/\[img\](.*?)\[\/img\]/is",$s,$matches)) {
+               $image = $matches[1];
+               logger('fix_private_photos: found photo ' . $image);
+               if(stristr($image ,$a->get_baseurl() . '/photo/')) {
+                       $i = basename($image);
+                       $i = str_replace('.jpg','',$i);
+                       $x = strpos($i,'-');
+                       if($x) {
+                               $res = substr($i,$x+1);
+                               $i = substr($i,0,$x);
+                               $r = q("SELECT * FROM `photo` WHERE `resource-id` = '%s' AND `scale` = %d AND `uid` = %d",
+                                       dbesc($i),
+                                       intval($res),
+                                       intval($uid)
+                               );
+                               if(count($r)) {
+                                       logger('replacing photo');
+                                       $s = str_replace($image, 'data:image/jpg;base64,' . base64_encode($r[0]['data']), $s);
+                               }
+                       }
+                       logger('fix_private_photos: replaced: ' . $s, LOGGER_DATA);
+               }       
+       }
+       return($s);
+}
+
+
+
+function item_getfeedtags($item) {
+       $ret = array();
+       $matches = false;
+       $cnt = preg_match_all('|\#\[url\=(.*?)\](.*?)\[\/url\]|',$item['tag'],$matches);
+       if($cnt) {
+               for($x = 0; $x < count($matches); $x ++) {
+                       if($matches[1][$x])
+                               $ret[] = array('#',$matches[1][$x], $matches[2][$x]);
+               }
+       }
+       $matches = false; 
+       $cnt = preg_match_all('|\@\[url\=(.*?)\](.*?)\[\/url\]|',$item['tag'],$matches);
+       if($cnt) {
+               for($x = 0; $x < count($matches); $x ++) {
+                       if($matches[1][$x])
+                               $ret[] = array('#',$matches[1][$x], $matches[2][$x]);
+               }
+       } 
+       return $ret;
+}
+
+function item_getfeedattach($item) {
+       $ret = '';
+       $arr = explode(',',$item['attach']);
+       if(count($arr)) {
+               foreach($arr as $r) {
+                       $matches = false;
+                       $cnt = preg_match('|\[attach\]href=\"(.*?)\" size=\"(.*?)\" type=\"(.*?)\" title=\"(.*?)\"\[\/attach\]|',$r,$matches);
+                       if($cnt) {
+                               $ret .= '<link rel="enclosure" href="' . xmlify($matches[1]) . '" type="' . xmlify($matches[3]) . '" ';
+                               if(intval($matches[2]))
+                                       $ret .= 'size="' . intval($matches[2]) . '" ';
+                               if($matches[4] !== ' ')
+                                       $ret .= 'title="' . xmlify(trim($matches[4])) . '" ';
+                               $ret .= ' />' . "\r\n";
+                       }
+               }
+       }
+       return $ret;
+}
+
+
        
+function item_expire($uid,$days) {
+
+       if((! $uid) || (! $days))
+               return;
+
+       $r = q("SELECT * FROM `item` 
+               WHERE `uid` = %d 
+               AND `created` < UTC_TIMESTAMP() - INTERVAL %d DAY 
+               AND `id` = `parent` 
+               AND `deleted` = 0",
+               intval($uid),
+               intval($days)
+       );
+
+       if(! count($r))
+               return;
+       logger('expire: # items=' . count($r) );
+
+       foreach($r as $item) {
+
+               // Only expire posts, not photos and photo comments
+
+               if(strlen($item['resource-id']))
+                       continue;
+
+               $r = q("UPDATE `item` SET `deleted` = 1, `edited` = '%s', `changed` = '%s' WHERE `id` = %d LIMIT 1",
+                       dbesc(datetime_convert()),
+                       dbesc(datetime_convert()),
+                       intval($item['id'])
+               );
+
+               // kill the kids
+
+               $r = q("UPDATE `item` SET `deleted` = 1, `edited` = '%s', `changed` = '%s' WHERE `parent-uri` = '%s' AND `uid` = %d ",
+                       dbesc(datetime_convert()),
+                       dbesc(datetime_convert()),
+                       dbesc($item['parent-uri']),
+                       intval($item['uid'])
+               );
+
+       }
+
+       proc_run('php',"include/notifier.php","expire","$uid");
+
+}
+