]> git.mxchange.org Git - friendica.git/blobdiff - src/Model/Item.php
Prevent multiple replacements for the same emoji in Protocol\ActivityPub\Processor...
[friendica.git] / src / Model / Item.php
index d18e96aa1618e76bf4b433ba4e9f9e9d2240ed5b..834a1c218d206437764143961ae5781eda2fe1e2 100644 (file)
@@ -29,7 +29,7 @@ use Friendica\Core\Protocol;
 use Friendica\Core\Renderer;
 use Friendica\Core\Session;
 use Friendica\Core\System;
-use Friendica\Core\Tag;
+use Friendica\Model\Tag;
 use Friendica\Core\Worker;
 use Friendica\Database\DBA;
 use Friendica\Database\DBStructure;
@@ -43,8 +43,8 @@ use Friendica\Util\Map;
 use Friendica\Util\Network;
 use Friendica\Util\Strings;
 use Friendica\Worker\Delivery;
-use Text_LanguageDetect;
 use Friendica\Repository\PermissionSet as RepPermissionSet;
+use LanguageDetection\Language;
 
 class Item
 {
@@ -58,17 +58,31 @@ class Item
        const PT_DOCUMENT = 19;
        const PT_EVENT = 32;
        const PT_TAG = 64;
+       const PT_TO = 65;
+       const PT_CC = 66;
+       const PT_BTO = 67;
+       const PT_BCC = 68;
+       const PT_FOLLOWER = 69;
+       const PT_ANNOUNCEMENT = 70;
+       const PT_COMMENT = 71;
+       const PT_STORED = 72;
+       const PT_GLOBAL = 73;
+       const PT_RELAY = 74;
+       const PT_FETCHED = 75;
        const PT_PERSONAL_NOTE = 128;
 
+       const LOCK_INSERT = 'item-insert';
+
        // Field list that is used to display the items
        const DISPLAY_FIELDLIST = [
                'uid', 'id', 'parent', 'uri-id', 'uri', 'thr-parent', 'parent-uri', 'guid', 'network', 'gravity',
                'commented', 'created', 'edited', 'received', 'verb', 'object-type', 'postopts', 'plink',
-               'wall', 'private', 'starred', 'origin', 'title', 'body', 'file', 'attach', 'language',
+               'wall', 'private', 'starred', 'origin', 'title', 'body', 'file', 'language',
                'content-warning', 'location', 'coord', 'app', 'rendered-hash', 'rendered-html', 'object',
                'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'item_id',
                'author-id', 'author-link', 'author-name', 'author-avatar', 'author-network',
                'owner-id', 'owner-link', 'owner-name', 'owner-avatar', 'owner-network',
+               'causer-id', 'causer-link', 'causer-name', 'causer-avatar', 'causer-contact-type',
                'contact-id', 'contact-uid', 'contact-link', 'contact-name', 'contact-avatar',
                'writable', 'self', 'cid', 'alias', 'pinned',
                'event-id', 'event-created', 'event-edited', 'event-start', 'event-finish',
@@ -81,7 +95,7 @@ class Item
        const DELIVER_FIELDLIST = ['uid', 'id', 'parent', 'uri-id', 'uri', 'thr-parent', 'parent-uri', 'guid',
                        'parent-guid', 'created', 'edited', 'verb', 'object-type', 'object', 'target',
                        'private', 'title', 'body', 'location', 'coord', 'app',
-                       'attach', 'deleted', 'extid', 'post-type', 'gravity',
+                       'deleted', 'extid', 'post-type', 'gravity',
                        'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid',
                        'author-id', 'author-link', 'owner-link', 'contact-uid',
                        'signed_text', 'signature', 'signer', 'network'];
@@ -92,21 +106,21 @@ class Item
                        'object-type', 'object', 'target-type', 'target', 'plink'];
 
        // Field list for "item-content" table that is not present in the "item" table
-       const CONTENT_FIELDLIST = ['language'];
+       const CONTENT_FIELDLIST = ['language', 'raw-body'];
 
        // All fields in the item table
        const ITEM_FIELDLIST = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent',
                        'guid', 'uri-id', 'parent-uri-id', 'thr-parent-id', 'vid',
                        'contact-id', 'type', 'wall', 'gravity', 'extid', 'icid', 'psid',
                        'created', 'edited', 'commented', 'received', 'changed', 'verb',
-                       'postopts', 'plink', 'resource-id', 'event-id', 'attach', 'inform',
+                       'postopts', 'plink', 'resource-id', 'event-id', 'inform',
                        'file', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'post-type',
                        'private', 'pubmail', 'moderated', 'visible', 'starred', 'bookmark',
                        'unseen', 'deleted', 'origin', 'forum_mode', 'mention', 'global', 'network',
                        'title', 'content-warning', 'body', 'location', 'coord', 'app',
                        'rendered-hash', 'rendered-html', 'object-type', 'object', 'target-type', 'target',
                        'author-id', 'author-link', 'author-name', 'author-avatar', 'author-network',
-                       'owner-id', 'owner-link', 'owner-name', 'owner-avatar'];
+                       'owner-id', 'owner-link', 'owner-name', 'owner-avatar', 'causer-id'];
 
        // List of all verbs that don't need additional content data.
        // Never reorder or remove entries from this list. Just add new ones at the end, if needed.
@@ -202,19 +216,7 @@ class Item
                        return [];
                }
 
-               if (empty($condition) || !is_array($condition)) {
-                       $condition = ['iid' => $pinned];
-               } else {
-                       reset($condition);
-                       $first_key = key($condition);
-                       if (!is_int($first_key)) {
-                               $condition['iid'] = $pinned;
-                       } else {
-                               $values_string = substr(str_repeat("?, ", count($pinned)), 0, -2);
-                               $condition[0] = '(' . $condition[0] . ") AND `iid` IN (" . $values_string . ")";
-                               $condition = array_merge($condition, $pinned);
-                       }
-               }
+               $condition = DBA::mergeConditions(['iid' => $pinned], $condition);
 
                return self::selectThreadForUser($uid, $selected, $condition, $params);
        }
@@ -654,10 +656,10 @@ class Item
                $fields = [];
 
                $fields['item'] = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent',
-                       'guid', 'uri-id', 'parent-uri-id', 'thr-parent-id', 'vid',
+                       'guid', 'uri-id', 'parent-uri-id', 'thr-parent-id', 'vid', 'causer-id',
                        'contact-id', 'owner-id', 'author-id', 'type', 'wall', 'gravity', 'extid',
                        'created', 'edited', 'commented', 'received', 'changed', 'psid',
-                       'resource-id', 'event-id', 'attach', 'post-type', 'file',
+                       'resource-id', 'event-id', 'post-type', 'file',
                        'private', 'pubmail', 'moderated', 'visible', 'starred', 'bookmark',
                        'unseen', 'deleted', 'origin', 'forum_mode', 'mention', 'global',
                        'id' => 'item_id', 'network', 'icid',
@@ -682,6 +684,10 @@ class Item
                $fields['owner'] = ['url' => 'owner-link', 'name' => 'owner-name', 'addr' => 'owner-addr',
                        'thumb' => 'owner-avatar', 'nick' => 'owner-nick', 'network' => 'owner-network'];
 
+               $fields['causer'] = ['url' => 'causer-link', 'name' => 'causer-name', 'addr' => 'causer-addr',
+                       'thumb' => 'causer-avatar', 'nick' => 'causer-nick', 'network' => 'causer-network',
+                       'contact-type' => 'causer-contact-type'];
+
                $fields['contact'] = ['url' => 'contact-link', 'name' => 'contact-name', 'thumb' => 'contact-avatar',
                        'writable', 'self', 'id' => 'cid', 'alias', 'uid' => 'contact-uid',
                        'photo', 'name-date', 'uri-date', 'avatar-date', 'thumb', 'dfrn-id'];
@@ -770,6 +776,9 @@ class Item
                                $joins .= " LEFT JOIN `contact` AS `owner` ON `owner`.`id` = $master_table.`owner-id`";
                        }
                }
+               if (strpos($sql_commands, "`causer`.") !== false) {
+                       $joins .= " LEFT JOIN `contact` AS `causer` ON `causer`.`id` = `item`.`causer-id`";
+               }
 
                if (strpos($sql_commands, "`group_member`.") !== false) {
                        $joins .= " STRAIGHT_JOIN `group_member` ON `group_member`.`contact-id` = $master_table.`contact-id`";
@@ -959,6 +968,14 @@ class Item
 
                while ($item = DBA::fetch($items)) {
                        if (empty($content_fields['verb']) || !in_array($content_fields['verb'], self::ACTIVITIES)) {
+                               if (!empty($content_fields['body'])) {
+                                       $content_fields['raw-body'] = trim($content_fields['raw-body'] ?? $content_fields['body']);
+               
+                                       // Remove all media attachments from the body and store them in the post-media table
+                                       $content_fields['raw-body'] = Post\Media::insertFromBody($item['uri-id'], $content_fields['raw-body']);
+                                       $content_fields['raw-body'] = self::setHashtags($content_fields['raw-body']);
+                               }
+               
                                self::updateContent($content_fields, ['uri-id' => $item['uri-id']]);
 
                                if (empty($item['icid'])) {
@@ -985,6 +1002,10 @@ class Item
                                }
                        }
 
+                       if (!empty($fields['attach'])) {
+                               Post\Media::insertFromAttachment($item['uri-id'], $fields['attach']);
+                       }
+
                        Post\DeliveryData::update($item['uri-id'], $delivery_data);
 
                        self::updateThread($item['id']);
@@ -1066,7 +1087,7 @@ class Item
                Logger::info('Mark item for deletion by id', ['id' => $item_id, 'callstack' => System::callstack()]);
                // locate item to be deleted
                $fields = ['id', 'uri', 'uri-id', 'uid', 'parent', 'parent-uri', 'origin',
-                       'deleted', 'file', 'resource-id', 'event-id', 'attach',
+                       'deleted', 'file', 'resource-id', 'event-id',
                        'verb', 'object-type', 'object', 'target', 'contact-id',
                        'icid', 'psid', 'gravity'];
                $item = self::selectFirst($fields, ['id' => $item_id]);
@@ -1123,10 +1144,9 @@ class Item
                }
 
                // If item has attachments, drop them
-               /// @TODO: this should first check if attachment is used elsewhere
-               foreach (explode(",", $item['attach']) as $attach) {
-                       preg_match("|attach/(\d+)|", $attach, $matches);
-                       if (is_array($matches) && count($matches) > 1) {
+               $attachments = Post\Media::getByURIId($item['uri-id'], [Post\Media::DOCUMENT]);
+               foreach($attachments as $attachment) {
+                       if (preg_match("|attach/(\d+)|", $attachment['url'], $matches)) {
                                Attach::delete(['id' => $matches[1], 'uid' => $item['uid']]);
                        }
                }
@@ -1671,7 +1691,7 @@ class Item
                $item['deny_gid']      = trim($item['deny_gid'] ?? '');
                $item['private']       = intval($item['private'] ?? self::PUBLIC);
                $item['body']          = trim($item['body'] ?? '');
-               $item['attach']        = trim($item['attach'] ?? '');
+               $item['raw-body']      = trim($item['raw-body'] ?? $item['body']);
                $item['app']           = trim($item['app'] ?? '');
                $item['origin']        = intval($item['origin'] ?? 0);
                $item['postopts']      = trim($item['postopts'] ?? '');
@@ -1692,10 +1712,10 @@ class Item
 
                $item['plink'] = ($item['plink'] ?? '') ?: DI::baseUrl() . '/display/' . urlencode($item['guid']);
 
-               $item['language'] = self::getLanguage($item);
-
                $item['gravity'] = self::getGravity($item);
 
+               $item['language'] = self::getLanguage($item);
+
                $default = ['url' => $item['author-link'], 'name' => $item['author-name'],
                        'photo' => $item['author-avatar'], 'network' => $item['network']];
                $item['author-id'] = ($item['author-id'] ?? 0) ?: Contact::getIdForURL($item['author-link'], 0, null, $default);
@@ -1704,6 +1724,11 @@ class Item
                        'photo' => $item['owner-avatar'], 'network' => $item['network']];
                $item['owner-id'] = ($item['owner-id'] ?? 0) ?: Contact::getIdForURL($item['owner-link'], 0, null, $default);
 
+               $actor = ($item['gravity'] == GRAVITY_PARENT) ? $item['owner-id'] : $item['author-id'];
+               if (!$item['origin'] && ($item['uid'] != 0) && Contact::isSharing($actor, $item['uid'])) {
+                       $item['post-type'] = self::PT_FOLLOWER;
+               }
+
                // Ensure that there is an avatar cache
                Contact::checkAvatarCache($item['author-id']);
                Contact::checkAvatarCache($item['owner-id']);
@@ -1715,8 +1740,7 @@ class Item
                        return 0;
                }
 
-               // We don't store the causer, we only have it here for the checks in the function above
-               unset($item['causer-id']);
+               // We don't store the causer link, only the id
                unset($item['causer-link']);
 
                // We don't store these fields anymore in the item table
@@ -1805,9 +1829,17 @@ class Item
                        self::setOwnerforResharedItem($item);
                }
 
+               // Remove all media attachments from the body and store them in the post-media table
+               $item['raw-body'] = Post\Media::insertFromBody($item['uri-id'], $item['raw-body']);
+               $item['raw-body'] = self::setHashtags($item['raw-body']);
+
                // Check for hashtags in the body and repair or add hashtag links
                $item['body'] = self::setHashtags($item['body']);
 
+               if (!empty($item['attach'])) {
+                       Post\Media::insertFromAttachment($item['uri-id'], $item['attach']);
+               }
+
                // Fill the cache field
                self::putInCache($item);
 
@@ -1817,15 +1849,17 @@ class Item
                        $notify_type = Delivery::POST;
                }
 
-               $like_no_comment = DI::config()->get('system', 'like_no_comment');
-
-               DBA::transaction();
-
                if (!in_array($item['verb'], self::ACTIVITIES)) {
                        $item['icid'] = self::insertContent($item);
+                       if (empty($item['icid'])) {
+                               // This shouldn't happen
+                               Logger::warning('No content stored, quitting', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'causer-id' => ($item['causer-id'] ?? 0), 'post-type' => $item['post-type'], 'network' => $item['network']]);
+                               return 0;
+                       }
                }
 
                $body = $item['body'];
+               $verb = $item['verb'];
 
                // We just remove everything that is content
                foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) {
@@ -1838,7 +1872,7 @@ class Item
 
                // Diaspora signature
                if (!empty($item['diaspora_signed_text'])) {
-                       DBA::insert('diaspora-interaction', ['uri-id' => $item['uri-id'], 'interaction' => $item['diaspora_signed_text']], true);
+                       DBA::replace('diaspora-interaction', ['uri-id' => $item['uri-id'], 'interaction' => $item['diaspora_signed_text']]);
                }
 
                unset($item['diaspora_signed_text']);
@@ -1871,47 +1905,33 @@ class Item
                        }
                }
 
-               $ret = DBA::insert('item', $item);
+               if (DI::lock()->acquire(self::LOCK_INSERT, 0)) {
+                       $condition = ['uri-id' => $item['uri-id'], 'uid' => $item['uid'], 'network' => $item['network']];
+                       if (DBA::exists('item', $condition)) {
+                               DI::lock()->release(self::LOCK_INSERT);
+                               Logger::notice('Item is already inserted - aborting', $condition);
+                               return 0;
+                       }
 
-               // When the item was successfully stored we fetch the ID of the item.
-               if (DBA::isResult($ret)) {
+                       $result = DBA::insert('item', $item);
+
+                       // When the item was successfully stored we fetch the ID of the item.
                        $current_post = DBA::lastInsertId();
+                       DI::lock()->release(self::LOCK_INSERT);
                } else {
-                       // This can happen - for example - if there are locking timeouts.
-                       DBA::rollback();
-
-                       // Store the data into a spool file so that we can try again later.
-                       self::spool($orig_item);
-                       return 0;
-               }
-
-               if ($current_post == 0) {
-                       // This is one of these error messages that never should occur.
-                       Logger::log("couldn't find created item - we better quit now.");
-                       DBA::rollback();
-                       return 0;
+                       Logger::warning('Item lock had not been acquired');
+                       $result = false;
+                       $current_post = 0;
                }
 
-               // How much entries have we created?
-               // We wouldn't need this query when we could use an unique index - but MySQL has length problems with them.
-               $entries = DBA::count('item', ['uri' => $item['uri'], 'uid' => $item['uid'], 'network' => $item['network']]);
-
-               if ($entries > 1) {
-                       // There are duplicates. We delete our just created entry.
-                       Logger::info('Delete duplicated item', ['id' => $current_post, 'uri' => $item['uri'], 'uid' => $item['uid'], 'guid' => $item['guid']]);
-
-                       // Yes, we could do a rollback here - but we possibly are still having users with MyISAM.
-                       DBA::delete('item', ['id' => $current_post]);
-                       DBA::commit();
-                       return 0;
-               } elseif ($entries == 0) {
-                       // This really should never happen since we quit earlier if there were problems.
-                       Logger::log("Something is terribly wrong. We haven't found our created entry.");
-                       DBA::rollback();
+               if (empty($current_post) || !DBA::isResult($result)) {
+                       // On failure store the data into a spool file so that the "SpoolPost" worker can try again later.
+                       Logger::warning('Could not store item. it will be spooled', ['result' => $result, 'id' => $current_post]);
+                       self::spool($orig_item);
                        return 0;
                }
 
-               Logger::log('created item '.$current_post);
+               Logger::notice('created item', ['id' => $current_post, 'uid' => $item['uid'], 'network' => $item['network'], 'uri-id' => $item['uri-id'], 'guid' => $item['guid']]);
 
                if (!$parent_id || ($item['parent-uri'] === $item['uri'])) {
                        $parent_id = $current_post;
@@ -1924,8 +1944,15 @@ class Item
                $item['parent'] = $parent_id;
 
                // update the commented timestamp on the parent
-               // Only update "commented" if it is really a comment
-               if (($item['gravity'] != GRAVITY_ACTIVITY) || !$like_no_comment) {
+               if (DI::config()->get('system', 'like_no_comment')) {
+                       // Update when it is a comment
+                       $update_commented = in_array($item['gravity'], [GRAVITY_PARENT, GRAVITY_COMMENT]);
+               } else {
+                       // Update when it isn't a follow or tag verb
+                       $update_commented = !in_array($verb, [Activity::FOLLOW, Activity::TAG]);
+               }
+
+               if ($update_commented) {
                        DBA::update('item', ['commented' => DateTimeFormat::utcNow(), 'changed' => DateTimeFormat::utcNow()], ['id' => $parent_id]);
                } else {
                        DBA::update('item', ['changed' => DateTimeFormat::utcNow()], ['id' => $parent_id]);
@@ -1936,7 +1963,6 @@ class Item
                } else {
                        self::updateThread($parent_id);
                }
-               DBA::commit();
 
                // In that function we check if this is a forum post. Additionally we delete the item under certain circumstances
                if (self::tagDeliver($item['uid'], $current_post)) {
@@ -2006,10 +2032,10 @@ class Item
         */
        private static function setOwnerforResharedItem(array $item)
        {
-               $parent = self::selectFirst(['id', 'owner-id', 'author-id', 'author-link', 'origin'],
-                       ['uri-id' => $item['parent-uri-id'], 'uid' => $item['uid']]);
+               $parent = self::selectFirst(['id', 'causer-id', 'owner-id', 'author-id', 'author-link', 'origin', 'post-type'],
+                       ['uri-id' => $item['thr-parent-id'], 'uid' => $item['uid']]);
                if (!DBA::isResult($parent)) {
-                       Logger::error('Parent not found', ['uri-id' => $item['parent-uri-id'], 'uid' => $item['uid']]);
+                       Logger::error('Parent not found', ['uri-id' => $item['thr-parent-id'], 'uid' => $item['uid']]);
                        return;
                }
 
@@ -2019,19 +2045,29 @@ class Item
                        return;
                }
 
-               if ($author['contact-type'] != Contact::TYPE_COMMUNITY) {
-                       logger::info('The resharer is no forum: quit', ['resharer' => $item['author-id'], 'owner' => $parent['owner-id'], 'author' => $parent['author-id'], 'uid' => $item['uid']]);
+               $cid = Contact::getIdForURL($author['url'], $item['uid']);
+               if (empty($cid) || !Contact::isSharing($cid, $item['uid'])) {
+                       Logger::info('The resharer is not a following contact: quit', ['resharer' => $author['url'], 'uid' => $item['uid']]);
                        return;
                }
 
-               $cid = Contact::getIdForURL($author['url'], $item['uid']);
-               if (empty($cid) || !Contact::isSharing($cid, $item['uid'])) {
-                       logger::info('The resharer is not a following contact: quit', ['resharer' => $author['url'], 'uid' => $item['uid']]);
+               if ($author['contact-type'] != Contact::TYPE_COMMUNITY) {
+                       if ($parent['post-type'] == self::PT_ANNOUNCEMENT) {
+                               Logger::info('The parent is already marked as announced: quit', ['causer' => $parent['causer-id'], 'owner' => $parent['owner-id'], 'author' => $parent['author-id'], 'uid' => $item['uid']]);
+                               return;
+                       }
+
+                       if (Contact::isSharing($parent['owner-id'], $item['uid'])) {
+                               Logger::info('The resharer is no forum: quit', ['resharer' => $item['author-id'], 'owner' => $parent['owner-id'], 'author' => $parent['author-id'], 'uid' => $item['uid']]);
+                               return;
+                       }
+                       self::update(['post-type' => self::PT_ANNOUNCEMENT, 'causer-id' => $item['author-id']], ['id' => $parent['id']]);
+                       Logger::info('Set announcement post-type', ['uri-id' => $item['uri-id'], 'thr-parent-id' => $item['thr-parent-id'], 'uid' => $item['uid']]);
                        return;
                }
 
-               Item::update(['owner-id' => $item['author-id'], 'contact-id' => $cid], ['id' => $parent['id']]);
-               Logger::info('Change owner of the parent', ['uri-id' => $item['uri-id'], 'parent-uri-id' => $item['parent-uri-id'], 'uid' => $item['uid'], 'owner-id' => $item['author-id'], 'contact-id' => $cid]);
+               self::update(['owner-id' => $item['author-id'], 'contact-id' => $cid], ['id' => $parent['id']]);
+               Logger::info('Change owner of the parent', ['uri-id' => $item['uri-id'], 'thr-parent-id' => $item['thr-parent-id'], 'uid' => $item['uid'], 'owner-id' => $item['author-id'], 'contact-id' => $cid]);
        }
 
        /**
@@ -2077,24 +2113,16 @@ class Item
                $item_content = DBA::selectFirst('item-content', ['id'], ['uri-id' => $item['uri-id']]);
                if (DBA::isResult($item_content)) {
                        $icid = $item_content['id'];
-                       Logger::info('Content found', ['icid' => $icid, 'uri' => $item['uri']]);
+                       Logger::info('Existing content found', ['icid' => $icid, 'uri' => $item['uri']]);
                        return $icid;
                }
 
-               DBA::insert('item-content', $fields, true);
-               $icid = DBA::lastInsertId();
-               if ($icid != 0) {
-                       Logger::info('Content inserted', ['icid' => $icid, 'uri' => $item['uri']]);
-                       return $icid;
-               }
+               DBA::replace('item-content', $fields);
 
-               // Possibly there can be timing issues. Then the same content could be inserted multiple times.
-               // Due to the indexes this doesn't happen, but "lastInsertId" will be empty in these situations.
-               // So we have to fetch the id manually. This is no bug and there is no data loss.
                $item_content = DBA::selectFirst('item-content', ['id'], ['uri-id' => $item['uri-id']]);
                if (DBA::isResult($item_content)) {
                        $icid = $item_content['id'];
-                       Logger::notice('Content inserted with empty lastInsertId', ['icid' => $icid, 'uri' => $item['uri']]);
+                       Logger::notice('Content inserted', ['icid' => $icid, 'uri' => $item['uri']]);
                        return $icid;
                }
 
@@ -2237,6 +2265,8 @@ class Item
                        return 0;
                }
 
+               $item['post-type'] = self::PT_STORED;
+
                $item = array_merge($item, $fields);
 
                $stored = self::storeForUser($item, $uid);
@@ -2320,7 +2350,7 @@ class Item
         * @param integer $itemid Item ID that should be added
         * @throws \Exception
         */
-       public static function addShadow($itemid)
+       private static function addShadow($itemid)
        {
                $fields = ['uid', 'private', 'moderated', 'visible', 'deleted', 'network', 'uri'];
                $condition = ['id' => $itemid, 'parent' => [0, $itemid]];
@@ -2362,6 +2392,7 @@ class Item
                        unset($item['starred']);
                        unset($item['postopts']);
                        unset($item['inform']);
+                       unset($item['post-type']);
                        if ($item['uri'] == $item['parent-uri']) {
                                $item['contact-id'] = $item['owner-id'];
                        } else {
@@ -2382,7 +2413,7 @@ class Item
         * @param integer $itemid Item ID that should be added
         * @throws \Exception
         */
-       public static function addShadowPost($itemid)
+       private static function addShadowPost($itemid)
        {
                $item = self::selectFirst(self::ITEM_FIELDLIST, ['id' => $itemid]);
                if (!DBA::isResult($item)) {
@@ -2424,6 +2455,7 @@ class Item
                unset($item['starred']);
                unset($item['postopts']);
                unset($item['inform']);
+               unset($item['post-type']);
                $item['contact-id'] = Contact::getIdForURL($item['author-link']);
 
                $public_shadow = self::insert($item, false, true);
@@ -2447,11 +2479,28 @@ class Item
         */
        private static function getLanguage(array $item)
        {
-               $naked_body = BBCode::toPlaintext($item['body'], false);
+               if (!in_array($item['gravity'], [GRAVITY_PARENT, GRAVITY_COMMENT])) {
+                       return '';
+               }
+
+               // Convert attachments to links
+               $naked_body = BBCode::removeAttachment($item['body']);
+
+               // Remove links and pictures
+               $naked_body = BBCode::removeLinks($naked_body);
 
-               $ld = new Text_LanguageDetect();
-               $ld->setNameMode(2);
-               $languages = $ld->detect($naked_body, 3);
+               // Convert the title and the body to plain text
+               $naked_body = trim($item['title'] . "\n" . BBCode::toPlaintext($naked_body));
+
+               // Remove possibly remaining links
+               $naked_body = preg_replace(Strings::autoLinkRegEx(), '', $naked_body);
+
+               if (empty($naked_body)) {
+                       return '';
+               }
+
+               $ld = new Language(DI::l10n()->getAvailableLanguages());
+               $languages = $ld->detect($naked_body)->limit(0, 3)->close();
                if (is_array($languages)) {
                        return json_encode($languages);
                }
@@ -2459,6 +2508,18 @@ class Item
                return '';
        }
 
+       public static function getLanguageMessage(array $item)
+       {
+               $iso639 = new \Matriphe\ISO639\ISO639;
+
+               $used_languages = '';
+               foreach (json_decode($item['language'], true) as $language => $reliability) {
+                       $used_languages .= $iso639->languageByCode1($language) . ' (' . $language . "): " . number_format($reliability, 5) . '\n';
+               }
+               $used_languages = DI::l10n()->t('Detected languages in this post:\n%s', $used_languages);
+               return $used_languages;
+       }
+
        /**
         * Creates an unique guid out of a given uri
         *
@@ -2672,11 +2733,11 @@ class Item
                }
 
                if (!$mention) {
-                       $tags = Tag::getByURIId($item_id, [Tag::MENTION, Tag::EXCLUSIVE_MENTION]);
-                       $foreach ($tags as $tag) {
+                       $tags = Tag::getByURIId($item['uri-id'], [Tag::MENTION, Tag::EXCLUSIVE_MENTION]);
+                       foreach ($tags as $tag) {
                                if (Strings::compareLink($link, $tag['url']) || Strings::compareLink($dlink, $tag['url'])) {
                                        $mention = true;
-                                       Logger::log('mention found in tag: ' . $tag['url']);
+                                       DI::logger()->info('mention found in tag.', ['url' => $tag['url']]);
                                }
                        }
                }
@@ -2794,14 +2855,19 @@ class Item
                        }
 
                        if ($contact['network'] != Protocol::FEED) {
+                               $old_uri_id = $datarray["uri-id"] ?? 0;
                                $datarray["guid"] = System::createUUID();
                                unset($datarray["plink"]);
                                $datarray["uri"] = self::newURI($contact['uid'], $datarray["guid"]);
+                               $datarray["uri-id"] = ItemURI::getIdByURI($datarray["uri"]);
                                $datarray["parent-uri"] = $datarray["uri"];
                                $datarray["thr-parent"] = $datarray["uri"];
                                $datarray["extid"] = Protocol::DFRN;
                                $urlpart = parse_url($datarray2['author-link']);
                                $datarray["app"] = $urlpart["host"];
+                               if (!empty($old_uri_id)) {
+                                       Post\Media::copy($old_uri_id, $datarray["uri-id"]);
+                               }
                        } else {
                                $datarray['private'] = self::PUBLIC;
                        }
@@ -3282,7 +3348,7 @@ class Item
                $item['iid'] = $itemid;
 
                if (!$onlyshadow) {
-                       $result = DBA::insert('thread', $item);
+                       $result = DBA::replace('thread', $item);
 
                        Logger::info('Add thread', ['item' => $itemid, 'result' => $result]);
                }
@@ -3337,6 +3403,37 @@ class Item
                }
        }
 
+       /**
+        * Fetch the SQL condition for the given user id
+        *
+        * @param integer $owner_id User ID for which the permissions should be fetched
+        * @return array condition
+        */
+       public static function getPermissionsConditionArrayByUserId(int $owner_id)
+       {
+               $local_user = local_user();
+               $remote_user = Session::getRemoteContactID($owner_id);
+
+               // default permissions - anonymous user
+               $condition = ["`private` != ?", self::PRIVATE];
+
+               if ($local_user && ($local_user == $owner_id)) {
+                       // Profile owner - everything is visible
+                       $condition = [];
+               } elseif ($remote_user) {
+                        // Authenticated visitor - fetch the matching permissionsets
+                       $set = PermissionSet::get($owner_id, $remote_user);
+                       if (!empty($set)) {
+                               $condition = ["(`private` != ? OR (`private` = ? AND `wall`
+                                       AND `psid` IN (" . implode(', ', array_fill(0, count($set), '?')) . ")))",
+                                       Item::PRIVATE, Item::PRIVATE];
+                               $condition = array_merge($condition, $set);
+                       }
+               }
+
+               return $condition;
+       }
+
        public static function getPermissionsSQLByUserId($owner_id)
        {
                $local_user = local_user();
@@ -3557,12 +3654,10 @@ class Item
 
                $as = '';
                $vhead = false;
-               $matches = [];
-               preg_match_all('|\[attach\]href=\"(.*?)\" length=\"(.*?)\" type=\"(.*?)\"(?: title=\"(.*?)\")?|', $item['attach'], $matches, PREG_SET_ORDER);
-               foreach ($matches as $mtch) {
-                       $mime = $mtch[3];
+               foreach (Post\Media::getByURIId($item['uri-id'], [Post\Media::DOCUMENT, Post\Media::TORRENT, Post\Media::UNKNOWN]) as $attachment) {
+                       $mime = $attachment['mimetype'];
 
-                       $the_url = Contact::magicLinkById($item['author-id'], $mtch[1]);
+                       $the_url = Contact::magicLinkById($item['author-id'], $attachment['url']);
 
                        if (strpos($mime, 'video') !== false) {
                                if (!$vhead) {
@@ -3570,11 +3665,9 @@ class Item
                                        DI::page()['htmlhead'] .= Renderer::replaceMacros(Renderer::getMarkupTemplate('videos_head.tpl'));
                                }
 
-                               $url_parts = explode('/', $the_url);
-                               $id = end($url_parts);
                                $as .= Renderer::replaceMacros(Renderer::getMarkupTemplate('video_top.tpl'), [
                                        '$video' => [
-                                               'id'     => $id,
+                                               'id'     => $item['author-id'],
                                                'title'  => DI::l10n()->t('View Video'),
                                                'src'    => $the_url,
                                                'mime'   => $mime,
@@ -3591,8 +3684,8 @@ class Item
                                $filesubtype = 'unkn';
                        }
 
-                       $title = Strings::escapeHtml(trim(($mtch[4] ?? '') ?: $mtch[1]));
-                       $title .= ' ' . $mtch[2] . ' ' . DI::l10n()->t('bytes');
+                       $title = Strings::escapeHtml(trim(($attachment['description'] ?? '') ?: $attachment['url']));
+                       $title .= ' ' . ($attachment['size'] ?? 0) . ' ' . DI::l10n()->t('bytes');
 
                        $icon = '<div class="attachtype icon s22 type-' . $filetype . ' subtype-' . $filesubtype . '"></div>';
                        $as .= '<a href="' . strip_tags($the_url) . '" title="' . $title . '" class="attachlink" target="_blank" rel="noopener noreferrer" >' . $icon . '</a>';
@@ -3831,7 +3924,7 @@ class Item
                $uid = $item['uid'] ?? 0;
 
                // first try to fetch the item via the GUID. This will work for all reshares that had been created on this system
-               $shared_item = self::selectFirst(['title', 'body', 'attach'], ['guid' => $shared['guid'], 'uid' => [0, $uid]]);
+               $shared_item = self::selectFirst(['title', 'body'], ['guid' => $shared['guid'], 'uid' => [0, $uid]]);
                if (!DBA::isResult($shared_item)) {
                        if (empty($shared['link'])) {
                                return $item;
@@ -3844,7 +3937,7 @@ class Item
                                return $item;
                        }
 
-                       $shared_item = self::selectFirst(['title', 'body', 'attach'], ['id' => $id]);
+                       $shared_item = self::selectFirst(['title', 'body'], ['id' => $id]);
                        if (!DBA::isResult($shared_item)) {
                                return $item;
                        }