From d3a2ed85fe425cffc103030a6027c9b91f9afc69 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 15 Jul 2018 20:36:20 +0200 Subject: [PATCH] Next item structure works (#5380) * Use "LEFT JOIN" to always fetch the item. Needed for update routines. * New conversion routine that now covers every item * Post update is now activated * We now use a hash based upon RIPEMD-320 for content and activity * The hash doesn't contain the plink anymore * Legacy item fields are now "null"able * New hash function for a server unique item hash * Introduction of the legacy mode (usage of old item fields) * Code simplification * We don't need the "uri" fields anymore in item-activity and item-content * Use the "created" and not the "received" date for the hash * Avoiding several notices * Some more warnings removed * Improved uri-hash / Likes on Diaspora are now getting a creation date * Corrected the post update version * Ensure an unique uri-hash * Don't delete orhaned item data at the moment * Partly reworked, due to strange behaviour * Some more parts reworked * Using the uri currently seems to be more reliable * Using the uri here as well * Use the hash values again * Grouped item fields in different categories * Notices again * use the gravity (we always should) * Added hint for disabled post updates * Notices ... * Issue #5337: Personal notes are displayed again * Use the gravity again --- boot.php | 6 +- database.sql | 103 ++++++++++---------- mod/friendica.php | 2 +- mod/notes.php | 8 +- mod/parse_url.php | 7 +- mod/settings.php | 8 +- src/Content/Text/BBCode.php | 4 +- src/Core/System.php | 5 +- src/Core/Worker.php | 2 +- src/Database/DBStructure.php | 120 ++++++++++++----------- src/Database/PostUpdate.php | 98 ++++++------------- src/Model/Item.php | 162 +++++++++++++++++++------------ src/Model/ItemContent.php | 4 +- src/Network/Probe.php | 12 +-- src/Protocol/Diaspora.php | 3 + src/Protocol/PortableContact.php | 25 +++-- src/Worker/CronJobs.php | 3 +- src/Worker/Delivery.php | 7 +- src/Worker/Expire.php | 4 +- src/Worker/Notifier.php | 4 +- src/Worker/SetItemContentID.php | 21 ---- 21 files changed, 313 insertions(+), 295 deletions(-) delete mode 100644 src/Worker/SetItemContentID.php diff --git a/boot.php b/boot.php index 218580a34d..ae33f7c414 100644 --- a/boot.php +++ b/boot.php @@ -41,7 +41,7 @@ define('FRIENDICA_PLATFORM', 'Friendica'); define('FRIENDICA_CODENAME', 'The Tazmans Flax-lily'); define('FRIENDICA_VERSION', '2018.08-dev'); define('DFRN_PROTOCOL_VERSION', '2.23'); -define('DB_UPDATE_VERSION', 1276); +define('DB_UPDATE_VERSION', 1277); define('NEW_UPDATE_ROUTINE_VERSION', 1170); /** @@ -929,6 +929,10 @@ function remote_user() */ function notice($s) { + if (empty($_SESSION)) { + return; + } + $a = get_app(); if (!x($_SESSION, 'sysmsg')) { $_SESSION['sysmsg'] = []; diff --git a/database.sql b/database.sql index 15f5a2f352..78632bbb6a 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2018.08-dev (The Tazmans Flax-lily) --- DB_UPDATE_VERSION 1276 +-- DB_UPDATE_VERSION 1277 -- ------------------------------------------ @@ -455,68 +455,69 @@ CREATE TABLE IF NOT EXISTS `item` ( `id` int unsigned NOT NULL auto_increment, `guid` varchar(255) NOT NULL DEFAULT '' COMMENT 'A unique identifier for this item', `uri` varchar(255) NOT NULL DEFAULT '' COMMENT '', - `uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'Owner id which owns this copy of the item', - `contact-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'contact.id', - `type` varchar(20) NOT NULL DEFAULT '' COMMENT '', - `wall` boolean NOT NULL DEFAULT '0' COMMENT 'This item was posted to the wall of uid', - `gravity` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '', + `uri-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri', `parent` int unsigned NOT NULL DEFAULT 0 COMMENT 'item.id of the parent to this item if it is a reply of some form; otherwise this must be set to the id of this item', `parent-uri` varchar(255) NOT NULL DEFAULT '' COMMENT 'uri of the parent to this item', - `extid` varchar(255) NOT NULL DEFAULT '' COMMENT '', `thr-parent` varchar(255) NOT NULL DEFAULT '' COMMENT 'If the parent of this item is not the top-level item in the conversation, the uri of the immediate parent; otherwise set to parent-uri', `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Creation timestamp.', `edited` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last edit (default is created)', `commented` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last comment/reply to this item', `received` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'datetime', `changed` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date that something in the conversation changed, indicating clients should fetch the conversation again', + `gravity` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '', + `network` char(4) NOT NULL DEFAULT '' COMMENT 'Network from where the item comes from', `owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the owner of this item', - `owner-name` varchar(255) NOT NULL DEFAULT '' COMMENT 'Name of the owner of this item', - `owner-link` varchar(255) NOT NULL DEFAULT '' COMMENT 'Link to the profile page of the owner of this item', - `owner-avatar` varchar(255) NOT NULL DEFAULT '' COMMENT 'Link to the avatar picture of the owner of this item', `author-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the author of this item', - `author-name` varchar(255) NOT NULL DEFAULT '' COMMENT 'Name of the author of this item', - `author-link` varchar(255) NOT NULL DEFAULT '' COMMENT 'Link to the profile page of the author of this item', - `author-avatar` varchar(255) NOT NULL DEFAULT '' COMMENT 'Link to the avatar picture of the author of this item', `icid` int unsigned COMMENT 'Id of the item-content table entry that contains the whole item content', `iaid` int unsigned COMMENT 'Id of the item-activity table entry that contains the activity data', - `title` varchar(255) NOT NULL DEFAULT '' COMMENT 'item title', - `content-warning` varchar(255) NOT NULL DEFAULT '' COMMENT '', - `body` mediumtext COMMENT 'item body content', - `app` varchar(255) NOT NULL DEFAULT '' COMMENT 'application which generated this item', - `verb` varchar(100) NOT NULL DEFAULT '' COMMENT 'ActivityStreams verb', - `object-type` varchar(100) NOT NULL DEFAULT '' COMMENT 'ActivityStreams object type', - `object` text COMMENT 'JSON encoded object structure unless it is an implied object (normal post)', - `target-type` varchar(100) NOT NULL DEFAULT '' COMMENT 'ActivityStreams target type if applicable (URI)', - `target` text COMMENT 'JSON encoded target structure if used', - `postopts` text COMMENT 'External post connectors add their network name to this comma-separated string to identify that they should be delivered to these networks during delivery', - `plink` varchar(255) NOT NULL DEFAULT '' COMMENT 'permalink or URL to a displayable copy of the message at its source', - `resource-id` varchar(32) NOT NULL DEFAULT '' COMMENT 'Used to link other tables to items, it identifies the linked resource (e.g. photo) and if set must also set resource_type', - `event-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Used to link to the event.id', - `tag` mediumtext COMMENT '', - `attach` mediumtext COMMENT 'JSON structure representing attachments to this item', - `inform` mediumtext COMMENT '', - `file` mediumtext COMMENT '', - `location` varchar(255) NOT NULL DEFAULT '' COMMENT 'text location where this item originated', - `coord` varchar(255) NOT NULL DEFAULT '' COMMENT 'longitude/latitude pair representing location where this item originated', - `allow_cid` mediumtext COMMENT 'Access Control - list of allowed contact.id \'<19><78>\'', - `allow_gid` mediumtext COMMENT 'Access Control - list of allowed groups', - `deny_cid` mediumtext COMMENT 'Access Control - list of denied contact.id', - `deny_gid` mediumtext COMMENT 'Access Control - list of denied groups', + `extid` varchar(255) NOT NULL DEFAULT '' COMMENT '', + `global` boolean NOT NULL DEFAULT '0' COMMENT '', `private` boolean NOT NULL DEFAULT '0' COMMENT 'distribution is restricted', - `pubmail` boolean NOT NULL DEFAULT '0' COMMENT '', - `moderated` boolean NOT NULL DEFAULT '0' COMMENT '', - `visible` boolean NOT NULL DEFAULT '0' COMMENT '', - `starred` boolean NOT NULL DEFAULT '0' COMMENT 'item has been favourited', `bookmark` boolean NOT NULL DEFAULT '0' COMMENT 'item has been bookmarked', - `unseen` boolean NOT NULL DEFAULT '1' COMMENT 'item has not been seen', + `visible` boolean NOT NULL DEFAULT '0' COMMENT '', + `moderated` boolean NOT NULL DEFAULT '0' COMMENT '', `deleted` boolean NOT NULL DEFAULT '0' COMMENT 'item has been deleted', + `uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'Owner id which owns this copy of the item', + `contact-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'contact.id', + `wall` boolean NOT NULL DEFAULT '0' COMMENT 'This item was posted to the wall of uid', `origin` boolean NOT NULL DEFAULT '0' COMMENT 'item originated at this site', - `forum_mode` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '', + `pubmail` boolean NOT NULL DEFAULT '0' COMMENT '', + `starred` boolean NOT NULL DEFAULT '0' COMMENT 'item has been favourited', + `unseen` boolean NOT NULL DEFAULT '1' COMMENT 'item has not been seen', `mention` boolean NOT NULL DEFAULT '0' COMMENT 'The owner of this item was mentioned in it', - `network` char(4) NOT NULL DEFAULT '' COMMENT 'Network from where the item comes from', - `rendered-hash` varchar(32) NOT NULL DEFAULT '' COMMENT '', - `rendered-html` mediumtext COMMENT 'item.body converted to html', - `global` boolean NOT NULL DEFAULT '0' COMMENT '', + `forum_mode` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '', + `allow_cid` mediumtext COMMENT 'Access Control - list of allowed contact.id \'<19><78>\'', + `allow_gid` mediumtext COMMENT 'Access Control - list of allowed groups', + `deny_cid` mediumtext COMMENT 'Access Control - list of denied contact.id', + `deny_gid` mediumtext COMMENT 'Access Control - list of denied groups', + `postopts` text COMMENT 'External post connectors add their network name to this comma-separated string to identify that they should be delivered to these networks during delivery', + `inform` mediumtext COMMENT 'Additional receivers of this post', + `resource-id` varchar(32) NOT NULL DEFAULT '' COMMENT 'Used to link other tables to items, it identifies the linked resource (e.g. photo) and if set must also set resource_type', + `event-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Used to link to the event.id', + `attach` mediumtext COMMENT 'JSON structure representing attachments to this item', + `type` varchar(20) NOT NULL DEFAULT '' COMMENT '', + `file` mediumtext COMMENT 'Deprecated', + `location` varchar(255) COMMENT 'Deprecated', + `coord` varchar(255) COMMENT 'Deprecated', + `tag` mediumtext COMMENT 'Deprecated', + `plink` varchar(255) COMMENT 'Deprecated', + `title` varchar(255) COMMENT 'Deprecated', + `content-warning` varchar(255) COMMENT 'Deprecated', + `body` mediumtext COMMENT 'Deprecated', + `app` varchar(255) COMMENT 'Deprecated', + `verb` varchar(100) COMMENT 'Deprecated', + `object-type` varchar(100) COMMENT 'Deprecated', + `object` text COMMENT 'Deprecated', + `target-type` varchar(100) COMMENT 'Deprecated', + `target` text COMMENT 'Deprecated', + `author-name` varchar(255) COMMENT 'Deprecated', + `author-link` varchar(255) COMMENT 'Deprecated', + `author-avatar` varchar(255) COMMENT 'Deprecated', + `owner-name` varchar(255) COMMENT 'Deprecated', + `owner-link` varchar(255) COMMENT 'Deprecated', + `owner-avatar` varchar(255) COMMENT 'Deprecated', + `rendered-hash` varchar(32) COMMENT 'Deprecated', + `rendered-html` mediumtext COMMENT 'Deprecated', PRIMARY KEY(`id`), INDEX `guid` (`guid`(191)), INDEX `uri` (`uri`(191)), @@ -549,8 +550,8 @@ CREATE TABLE IF NOT EXISTS `item` ( -- CREATE TABLE IF NOT EXISTS `item-activity` ( `id` int unsigned NOT NULL auto_increment, - `uri` varchar(255) NOT NULL DEFAULT '' COMMENT '', - `uri-hash` char(80) NOT NULL DEFAULT '' COMMENT 'SHA-1 and RIPEMD-160 hash from uri', + `uri` varchar(255) COMMENT '', + `uri-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri', `activity` smallint unsigned NOT NULL DEFAULT 0 COMMENT '', PRIMARY KEY(`id`), UNIQUE INDEX `uri-hash` (`uri-hash`), @@ -562,8 +563,8 @@ CREATE TABLE IF NOT EXISTS `item-activity` ( -- CREATE TABLE IF NOT EXISTS `item-content` ( `id` int unsigned NOT NULL auto_increment, - `uri` varchar(255) NOT NULL DEFAULT '' COMMENT '', - `uri-plink-hash` char(80) NOT NULL DEFAULT '' COMMENT 'SHA-1 hash from uri and plink', + `uri` varchar(255) COMMENT '', + `uri-plink-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri', `title` varchar(255) NOT NULL DEFAULT '' COMMENT 'item title', `content-warning` varchar(255) NOT NULL DEFAULT '' COMMENT '', `body` mediumtext COMMENT 'item body content', @@ -591,8 +592,8 @@ CREATE TABLE IF NOT EXISTS `locks` ( `id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID', `name` varchar(128) NOT NULL DEFAULT '' COMMENT '', `locked` boolean NOT NULL DEFAULT '0' COMMENT '', - `expires` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'datetime of lock expiration', `pid` int unsigned NOT NULL DEFAULT 0 COMMENT 'Process ID', + `expires` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'datetime of cache expiration', PRIMARY KEY(`id`), INDEX `name_expires` (`name`,`expires`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT=''; diff --git a/mod/friendica.php b/mod/friendica.php index e75e9cebae..9d4bdd801c 100644 --- a/mod/friendica.php +++ b/mod/friendica.php @@ -42,7 +42,7 @@ function friendica_init(App $a) Config::load('feature_lock'); $locked_features = []; - if (is_array($a->config['feature_lock']) && count($a->config['feature_lock'])) { + if (!empty($a->config['feature_lock']) && count($a->config['feature_lock'])) { foreach ($a->config['feature_lock'] as $k => $v) { if ($k === 'config_loaded') { continue; diff --git a/mod/notes.php b/mod/notes.php index 99114add8c..553656406e 100644 --- a/mod/notes.php +++ b/mod/notes.php @@ -57,9 +57,9 @@ function notes_content(App $a, $update = false) $o .= status_editor($a, $x, $a->contact['id']); } - $condition = ["`uid` = ? AND `type` = 'note' AND `id` = `parent` AND NOT `wall` + $condition = ["`uid` = ? AND `type` = 'note' AND `gravity` = ? AND NOT `wall` AND `allow_cid` = ? AND `contact-id` = ?", - local_user(), '<' . $a->contact['id'] . '>', $a->contact['id']]; + local_user(), GRAVITY_PARENT, '<' . $a->contact['id'] . '>', $a->contact['id']]; $notes = dba::count('item', $condition); @@ -68,13 +68,13 @@ function notes_content(App $a, $update = false) $params = ['order' => ['created' => true], 'limit' => [$a->pager['start'], $a->pager['itemspage']]]; - $r = Item::selectForUser(local_user(), ['item_id'], $condition, $params); + $r = Item::selectForUser(local_user(), ['id'], $condition, $params); if (DBM::is_result($r)) { $parents_arr = []; while ($rr = Item::fetch($r)) { - $parents_arr[] = $rr['item_id']; + $parents_arr[] = $rr['id']; } dba::close($r); diff --git a/mod/parse_url.php b/mod/parse_url.php index ea860f6d31..7a5442311f 100644 --- a/mod/parse_url.php +++ b/mod/parse_url.php @@ -67,8 +67,11 @@ function parse_url_content(App $a) { $hdrs = []; $h = explode("\n", $result["header"]); foreach ($h as $l) { - list($k,$v) = array_map("trim", explode(":", trim($l), 2)); - $hdrs[$k] = $v; + $header = array_map("trim", explode(":", trim($l), 2)); + if (count($header) == 2) { + list($k,$v) = $header; + $hdrs[$k] = $v; + } } if (array_key_exists("Content-Type", $hdrs)) { $type = $hdrs["Content-Type"]; diff --git a/mod/settings.php b/mod/settings.php index cd45cc5070..d7e8b7b459 100644 --- a/mod/settings.php +++ b/mod/settings.php @@ -26,12 +26,14 @@ use Friendica\Util\Temporal; function get_theme_config_file($theme) { $a = get_app(); - $base_theme = $a->theme_info['extends']; + if (!empty($a->theme_info['extends'])) { + $base_theme = $a->theme_info['extends']; + } if (file_exists("view/theme/$theme/config.php")) { return "view/theme/$theme/config.php"; } - if (file_exists("view/theme/$base_theme/config.php")) { + if (!empty($base_theme) && file_exists("view/theme/$base_theme/config.php")) { return "view/theme/$base_theme/config.php"; } return null; @@ -1155,7 +1157,7 @@ function settings_content(App $a) // Private/public post links for the non-JS ACL form $private_post = 1; - if ($_REQUEST['public']) { + if (!empty($_REQUEST['public']) && !$_REQUEST['public']) { $private_post = 0; } diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index 72d20da178..de3877d3f1 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -579,7 +579,7 @@ class BBCode extends BaseObject $return .= sprintf('
', $data["url"], self::proxyUrl($data["preview"], $simplehtml), $data["title"]); } - if (($data["type"] == "photo") && ($data["url"] != "") && ($data["image"] != "")) { + if (($data["type"] == "photo") && !empty($data["url"]) && !empty($data["image"])) { $return .= sprintf('', $data["url"], self::proxyUrl($data["image"], $simplehtml), $data["title"]); } else { $return .= sprintf('

%s

', $data['url'], $data['title']); @@ -613,7 +613,7 @@ class BBCode extends BaseObject return $data["text"] . $data["after"]; } - $title = htmlentities($data["title"], ENT_QUOTES, 'UTF-8', false); + $title = htmlentities(defaults($data, 'title', ''), ENT_QUOTES, 'UTF-8', false); $text = htmlentities($data["text"], ENT_QUOTES, 'UTF-8', false); if ($plaintext || (($title != "") && strstr($text, $title))) { $data["title"] = $data["url"]; diff --git a/src/Core/System.php b/src/Core/System.php index 98ae3da2d9..e3dc4e5870 100644 --- a/src/Core/System.php +++ b/src/Core/System.php @@ -212,7 +212,10 @@ EOT; */ public static function processID($prefix) { - return uniqid($prefix . ':' . str_pad(getmypid() . ':', 8, '0') . ':'); + // We aren't calling any other function here. + // Doing so could easily create an endless loop + $trailer = $prefix . ':' . getmypid() . ':'; + return substr($trailer . uniqid('') . mt_rand(), 0, 26); } /// @todo Move the following functions from boot.php diff --git a/src/Core/Worker.php b/src/Core/Worker.php index 5f19870897..0d51bb725d 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -352,7 +352,7 @@ class Worker $a->process_id = $old_process_id; unset($a->queue); - $duration = number_format(microtime(true) - $stamp, 3); + $duration = (microtime(true) - $stamp); self::$up_start = microtime(true); diff --git a/src/Database/DBStructure.php b/src/Database/DBStructure.php index 54e896b695..5c03c1afd6 100644 --- a/src/Database/DBStructure.php +++ b/src/Database/DBStructure.php @@ -536,26 +536,26 @@ class DBStructure private static function FieldCommand($parameters, $create = true) { $fieldstruct = $parameters["type"]; - if (!empty($parameters["Collation"])) { + if (isset($parameters["Collation"])) { $fieldstruct .= " COLLATE ".$parameters["Collation"]; } - if (!empty($parameters["not null"])) { + if (isset($parameters["not null"])) { $fieldstruct .= " NOT NULL"; } - if (!empty($parameters["default"])) { + if (isset($parameters["default"])) { if (strpos(strtolower($parameters["type"]),"int")!==false) { $fieldstruct .= " DEFAULT ".$parameters["default"]; } else { $fieldstruct .= " DEFAULT '".$parameters["default"]."'"; } } - if (!empty($parameters["extra"])) { + if (isset($parameters["extra"])) { $fieldstruct .= " ".$parameters["extra"]; } - if (!empty($parameters["comment"])) { + if (isset($parameters["comment"])) { $fieldstruct .= " COMMENT '".dbesc($parameters["comment"])."'"; } @@ -588,11 +588,11 @@ class DBStructure } } - if (!empty($structure["engine"])) { + if (isset($structure["engine"])) { $engine = " ENGINE=" . $structure["engine"]; } - if (!empty($structure["comment"])) { + if (isset($structure["comment"])) { $comment = " COMMENT='" . dbesc($structure["comment"]) . "'"; } @@ -1158,68 +1158,76 @@ class DBStructure "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "relation" => ["thread" => "iid"]], "guid" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "A unique identifier for this item"], "uri" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], - "uid" => ["type" => "mediumint unsigned", "not null" => "1", "default" => "0", "relation" => ["user" => "uid"], "comment" => "Owner id which owns this copy of the item"], - "contact-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "contact.id"], - "type" => ["type" => "varchar(20)", "not null" => "1", "default" => "", "comment" => ""], - "wall" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "This item was posted to the wall of uid"], - "gravity" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => ""], + "uri-hash" => ["type" => "varchar(80)", "not null" => "1", "default" => "", "comment" => "RIPEMD-128 hash from uri"], "parent" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["item" => "id"], "comment" => "item.id of the parent to this item if it is a reply of some form; otherwise this must be set to the id of this item"], "parent-uri" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "uri of the parent to this item"], - "extid" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], "thr-parent" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "If the parent of this item is not the top-level item in the conversation, the uri of the immediate parent; otherwise set to parent-uri"], "created" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Creation timestamp."], "edited" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Date of last edit (default is created)"], "commented" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Date of last comment/reply to this item"], "received" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "datetime"], "changed" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Date that something in the conversation changed, indicating clients should fetch the conversation again"], + "gravity" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => ""], + "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => "Network from where the item comes from"], "owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "Link to the contact table with uid=0 of the owner of this item"], - "owner-name" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Name of the owner of this item"], - "owner-link" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Link to the profile page of the owner of this item"], - "owner-avatar" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Link to the avatar picture of the owner of this item"], "author-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "Link to the contact table with uid=0 of the author of this item"], - "author-name" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Name of the author of this item"], - "author-link" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Link to the profile page of the author of this item"], - "author-avatar" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Link to the avatar picture of the author of this item"], "icid" => ["type" => "int unsigned", "relation" => ["item-content" => "id"], "comment" => "Id of the item-content table entry that contains the whole item content"], "iaid" => ["type" => "int unsigned", "relation" => ["item-activity" => "id"], "comment" => "Id of the item-activity table entry that contains the activity data"], - "title" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "item title"], - "content-warning" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], - "body" => ["type" => "mediumtext", "comment" => "item body content"], - "app" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "application which generated this item"], - "verb" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams verb"], - "object-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams object type"], - "object" => ["type" => "text", "comment" => "JSON encoded object structure unless it is an implied object (normal post)"], - "target-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams target type if applicable (URI)"], - "target" => ["type" => "text", "comment" => "JSON encoded target structure if used"], - "postopts" => ["type" => "text", "comment" => "External post connectors add their network name to this comma-separated string to identify that they should be delivered to these networks during delivery"], - "plink" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "permalink or URL to a displayable copy of the message at its source"], - "resource-id" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => "Used to link other tables to items, it identifies the linked resource (e.g. photo) and if set must also set resource_type"], - "event-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["event" => "id"], "comment" => "Used to link to the event.id"], - "tag" => ["type" => "mediumtext", "comment" => ""], - "attach" => ["type" => "mediumtext", "comment" => "JSON structure representing attachments to this item"], - "inform" => ["type" => "mediumtext", "comment" => ""], - "file" => ["type" => "mediumtext", "comment" => ""], - "location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "text location where this item originated"], - "coord" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "longitude/latitude pair representing location where this item originated"], - "allow_cid" => ["type" => "mediumtext", "comment" => "Access Control - list of allowed contact.id '<19><78>'"], - "allow_gid" => ["type" => "mediumtext", "comment" => "Access Control - list of allowed groups"], - "deny_cid" => ["type" => "mediumtext", "comment" => "Access Control - list of denied contact.id"], - "deny_gid" => ["type" => "mediumtext", "comment" => "Access Control - list of denied groups"], + "extid" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], + "global" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""], "private" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "distribution is restricted"], - "pubmail" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""], - "moderated" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""], - "visible" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""], - "starred" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "item has been favourited"], "bookmark" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "item has been bookmarked"], - "unseen" => ["type" => "boolean", "not null" => "1", "default" => "1", "comment" => "item has not been seen"], + "visible" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""], + "moderated" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""], "deleted" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "item has been deleted"], + // User specific fields. Eventually they will move to user-item + "uid" => ["type" => "mediumint unsigned", "not null" => "1", "default" => "0", "relation" => ["user" => "uid"], "comment" => "Owner id which owns this copy of the item"], + "contact-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "contact.id"], + "wall" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "This item was posted to the wall of uid"], "origin" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "item originated at this site"], - "forum_mode" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => ""], + "pubmail" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""], + "starred" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "item has been favourited"], + "unseen" => ["type" => "boolean", "not null" => "1", "default" => "1", "comment" => "item has not been seen"], "mention" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "The owner of this item was mentioned in it"], - "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => "Network from where the item comes from"], - "rendered-hash" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => ""], - "rendered-html" => ["type" => "mediumtext", "comment" => "item.body converted to html"], - "global" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""], + "forum_mode" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => ""], + // User specific fields. Should possible be replaced with something different + "allow_cid" => ["type" => "mediumtext", "comment" => "Access Control - list of allowed contact.id '<19><78>'"], + "allow_gid" => ["type" => "mediumtext", "comment" => "Access Control - list of allowed groups"], + "deny_cid" => ["type" => "mediumtext", "comment" => "Access Control - list of denied contact.id"], + "deny_gid" => ["type" => "mediumtext", "comment" => "Access Control - list of denied groups"], + "postopts" => ["type" => "text", "comment" => "External post connectors add their network name to this comma-separated string to identify that they should be delivered to these networks during delivery"], + "inform" => ["type" => "mediumtext", "comment" => "Additional receivers of this post"], + // It is to be decided whether these fields belong to the user or the structure + "resource-id" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => "Used to link other tables to items, it identifies the linked resource (e.g. photo) and if set must also set resource_type"], + "event-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["event" => "id"], "comment" => "Used to link to the event.id"], + // Will be replaced by the "attach" table + "attach" => ["type" => "mediumtext", "comment" => "JSON structure representing attachments to this item"], + // Seems to be only used for notes, but is filled at many places. + // Will be replaced with some general field that contain the values of "origin" and "wall" as well. + "type" => ["type" => "varchar(20)", "not null" => "1", "default" => "", "comment" => ""], + // Deprecated fields. Will be removed in upcoming versions + "file" => ["type" => "mediumtext", "comment" => "Deprecated"], + "location" => ["type" => "varchar(255)", "comment" => "Deprecated"], + "coord" => ["type" => "varchar(255)", "comment" => "Deprecated"], + "tag" => ["type" => "mediumtext", "comment" => "Deprecated"], + "plink" => ["type" => "varchar(255)", "comment" => "Deprecated"], + "title" => ["type" => "varchar(255)", "comment" => "Deprecated"], + "content-warning" => ["type" => "varchar(255)", "comment" => "Deprecated"], + "body" => ["type" => "mediumtext", "comment" => "Deprecated"], + "app" => ["type" => "varchar(255)", "comment" => "Deprecated"], + "verb" => ["type" => "varchar(100)", "comment" => "Deprecated"], + "object-type" => ["type" => "varchar(100)", "comment" => "Deprecated"], + "object" => ["type" => "text", "comment" => "Deprecated"], + "target-type" => ["type" => "varchar(100)", "comment" => "Deprecated"], + "target" => ["type" => "text", "comment" => "Deprecated"], + "author-name" => ["type" => "varchar(255)", "comment" => "Deprecated"], + "author-link" => ["type" => "varchar(255)", "comment" => "Deprecated"], + "author-avatar" => ["type" => "varchar(255)", "comment" => "Deprecated"], + "owner-name" => ["type" => "varchar(255)", "comment" => "Deprecated"], + "owner-link" => ["type" => "varchar(255)", "comment" => "Deprecated"], + "owner-avatar" => ["type" => "varchar(255)", "comment" => "Deprecated"], + "rendered-hash" => ["type" => "varchar(32)", "comment" => "Deprecated"], + "rendered-html" => ["type" => "mediumtext", "comment" => "Deprecated"], ], "indexes" => [ "PRIMARY" => ["id"], @@ -1253,8 +1261,8 @@ class DBStructure "comment" => "Activities for items", "fields" => [ "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "relation" => ["thread" => "iid"]], - "uri" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], - "uri-hash" => ["type" => "char(80)", "not null" => "1", "default" => "", "comment" => "SHA-1 and RIPEMD-160 hash from uri"], + "uri" => ["type" => "varchar(255)", "comment" => ""], + "uri-hash" => ["type" => "varchar(80)", "not null" => "1", "default" => "", "comment" => "RIPEMD-128 hash from uri"], "activity" => ["type" => "smallint unsigned", "not null" => "1", "default" => "0", "comment" => ""], ], "indexes" => [ @@ -1267,8 +1275,8 @@ class DBStructure "comment" => "Content for all posts", "fields" => [ "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "relation" => ["thread" => "iid"]], - "uri" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], - "uri-plink-hash" => ["type" => "char(80)", "not null" => "1", "default" => "", "comment" => "SHA-1 hash from uri and plink"], + "uri" => ["type" => "varchar(255)", "comment" => ""], + "uri-plink-hash" => ["type" => "varchar(80)", "not null" => "1", "default" => "", "comment" => "RIPEMD-128 hash from uri"], "title" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "item title"], "content-warning" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], "body" => ["type" => "mediumtext", "comment" => "item body content"], diff --git a/src/Database/PostUpdate.php b/src/Database/PostUpdate.php index 88bec31b74..d391d531d5 100644 --- a/src/Database/PostUpdate.php +++ b/src/Database/PostUpdate.php @@ -31,10 +31,7 @@ class PostUpdate if (!self::update1206()) { return; } - if (!self::update1274()) { - return; - } - if (!self::update1275()) { + if (!self::update1276()) { return; } } @@ -226,95 +223,62 @@ class PostUpdate } /** - * @brief update the "item-content" table + * @brief update the item related tables * * @return bool "true" when the job is done */ - private static function update1274() + private static function update1276() { // Was the script completed? - if (Config::get("system", "post_update_version") >= 1274) { + if (Config::get("system", "post_update_version") >= 1276) { return true; } - logger("Start", LOGGER_DEBUG); + $id = Config::get("system", "post_update_version_1276_id", 0); - $fields = ['id', 'title', 'content-warning', 'body', 'location', 'tag', 'file', - 'coord', 'app', 'rendered-hash', 'rendered-html', 'verb', - 'object-type', 'object', 'target-type', 'target', 'plink', - 'author-id', 'owner-id']; - - $condition = ["`icid` IS NULL"]; - $params = ['limit' => 10000]; - $items = Item::select($fields, $condition, $params); + logger("Start from item " . $id, LOGGER_DEBUG); - if (!DBM::is_result($items)) { - Config::set("system", "post_update_version", 1274); - logger("Done", LOGGER_DEBUG); - return true; - } + $fields = array_merge(Item::MIXED_CONTENT_FIELDLIST, ['network', 'author-id', 'owner-id', 'tag', 'file', + 'author-name', 'author-avatar', 'author-link', 'owner-name', 'owner-avatar', 'owner-link', 'id']); + $start_id = $id; $rows = 0; - + $condition = ["`id` > ?", $id]; + $params = ['order' => ['id'], 'limit' => 10000]; + $items = Item::select($fields, $condition, $params); while ($item = Item::fetch($items)) { - // Clearing the author and owner data if there is an id. - if ($item['author-id'] > 0) { - $item['author-name'] = ''; - $item['author-link'] = ''; - $item['author-avatar'] = ''; + $id = $item['id']; + + if (empty($item['author-id'])) { + $default = ['url' => $item['author-link'], 'name' => $item['author-name'], + 'photo' => $item['author-avatar'], 'network' => $item['network']]; + + $item['author-id'] = Contact::getIdForURL($item["author-link"], 0, false, $default); } - if ($item['owner-id'] > 0) { - $item['owner-name'] = ''; - $item['owner-link'] = ''; - $item['owner-avatar'] = ''; + if (empty($item['owner-id'])) { + $default = ['url' => $item['owner-link'], 'name' => $item['owner-name'], + 'photo' => $item['owner-avatar'], 'network' => $item['network']]; + + $item['owner-id'] = Contact::getIdForURL($item["owner-link"], 0, false, $default); } - Item::update($item, ['id' => $item['id']]); + Item::update($item, ['id' => $id]); + ++$rows; } dba::close($items); - logger("Processed rows: " . $rows, LOGGER_DEBUG); - return true; - } - /** - * @brief update the "item-activity" table - * - * @return bool "true" when the job is done - */ - private static function update1275() - { - // Was the script completed? - if (Config::get("system", "post_update_version") >= 1275) { - return true; - } - - logger("Start", LOGGER_DEBUG); - - $fields = ['id', 'verb']; + Config::set("system", "post_update_version_1276_id", $id); - $condition = ["`iaid` IS NULL AND NOT `icid` IS NULL AND `verb` IN (?, ?, ?, ?, ?)", - ACTIVITY_LIKE, ACTIVITY_DISLIKE, ACTIVITY_ATTEND, ACTIVITY_ATTENDNO, ACTIVITY_ATTENDMAYBE]; - - $params = ['limit' => 10000]; - $items = Item::select($fields, $condition, $params); + logger("Processed rows: " . $rows . " - last processed item: " . $id, LOGGER_DEBUG); - if (!DBM::is_result($items)) { - Config::set("system", "post_update_version", 1275); + if ($start_id == $id) { + Config::set("system", "post_update_version", 1276); logger("Done", LOGGER_DEBUG); return true; } - $rows = 0; - - while ($item = Item::fetch($items)) { - Item::update($item, ['id' => $item['id']]); - ++$rows; - } - dba::close($items); - - logger("Processed rows: " . $rows, LOGGER_DEBUG); - return true; + return false; } } diff --git a/src/Model/Item.php b/src/Model/Item.php index 670d5b4267..d6635407dc 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -78,6 +78,17 @@ class Item extends BaseObject // The item-activity table only stores the index and needs this array to know the matching activity. const ACTIVITIES = [ACTIVITY_LIKE, ACTIVITY_DISLIKE, ACTIVITY_ATTEND, ACTIVITY_ATTENDNO, ACTIVITY_ATTENDMAYBE]; + private static $legacy_mode = null; + + public static function isLegacyMode() + { + if (is_null(self::$legacy_mode)) { + self::$legacy_mode = (Config::get("system", "post_update_version") < 1276); + } + + return self::$legacy_mode; + } + /** * @brief returns an activity index from an activity string * @@ -156,11 +167,13 @@ class Item extends BaseObject // ---------------------- Transform item content data ---------------------- // Fetch data from the item-content table whenever there is content there - foreach (self::MIXED_CONTENT_FIELDLIST as $field) { - if (empty($row[$field]) && !empty($row['internal-item-' . $field])) { - $row[$field] = $row['internal-item-' . $field]; + if (self::isLegacyMode()) { + foreach (self::MIXED_CONTENT_FIELDLIST as $field) { + if (empty($row[$field]) && !empty($row['internal-item-' . $field])) { + $row[$field] = $row['internal-item-' . $field]; + } + unset($row['internal-item-' . $field]); } - unset($row['internal-item-' . $field]); } if (!empty($row['internal-iaid']) && array_key_exists('verb', $row)) { @@ -568,20 +581,20 @@ class Item extends BaseObject $joins .= sprintf("STRAIGHT_JOIN `contact` ON `contact`.`id` = $master_table.`contact-id` AND NOT `contact`.`blocked` AND ((NOT `contact`.`readonly` AND NOT `contact`.`pending` AND (`contact`.`rel` IN (%s, %s))) - OR `contact`.`self` OR (`item`.`id` != `item`.`parent`) OR `contact`.`uid` = 0) + OR `contact`.`self` OR `item`.`gravity` != %d OR `contact`.`uid` = 0) STRAIGHT_JOIN `contact` AS `author` ON `author`.`id` = $master_table.`author-id` AND NOT `author`.`blocked` STRAIGHT_JOIN `contact` AS `owner` ON `owner`.`id` = $master_table.`owner-id` AND NOT `owner`.`blocked` LEFT JOIN `user-item` ON `user-item`.`iid` = $master_table_key AND `user-item`.`uid` = %d", - CONTACT_IS_SHARING, CONTACT_IS_FRIEND, intval($uid)); + CONTACT_IS_SHARING, CONTACT_IS_FRIEND, GRAVITY_PARENT, intval($uid)); } else { if (strpos($sql_commands, "`contact`.") !== false) { - $joins .= "STRAIGHT_JOIN `contact` ON `contact`.`id` = $master_table.`contact-id`"; + $joins .= "LEFT JOIN `contact` ON `contact`.`id` = $master_table.`contact-id`"; } if (strpos($sql_commands, "`author`.") !== false) { - $joins .= " STRAIGHT_JOIN `contact` AS `author` ON `author`.`id` = $master_table.`author-id`"; + $joins .= " LEFT JOIN `contact` AS `author` ON `author`.`id` = $master_table.`author-id`"; } if (strpos($sql_commands, "`owner`.") !== false) { - $joins .= " STRAIGHT_JOIN `contact` AS `owner` ON `owner`.`id` = $master_table.`owner-id`"; + $joins .= " LEFT JOIN `contact` AS `owner` ON `owner`.`id` = $master_table.`owner-id`"; } } @@ -645,7 +658,7 @@ class Item extends BaseObject foreach ($fields as $table => $table_fields) { foreach ($table_fields as $field => $select) { if (empty($selected) || in_array($select, $selected)) { - if (in_array($select, self::MIXED_CONTENT_FIELDLIST)) { + if (self::isLegacyMode() && in_array($select, self::MIXED_CONTENT_FIELDLIST)) { $selection[] = "`item`.`".$select."` AS `internal-item-" . $select . "`"; } if (is_int($field)) { @@ -685,6 +698,19 @@ class Item extends BaseObject return $query; } + /** + * @brief Generate a server unique item hash for linking between the item tables + * + * @param string $uri Item URI + * @param date $created Item creation date + * + * @return string the item hash + */ + private static function itemHash($uri, $created) + { + return round(strtotime($created) / 100) . hash('ripemd128', $uri); + } + /** * @brief Update existing item entries * @@ -710,13 +736,13 @@ class Item extends BaseObject // We cannot simply expand the condition to check for origin entries // The condition needn't to be a simple array but could be a complex condition. // And we have to execute this query before the update to ensure to fetch the same data. - $items = dba::select('item', ['id', 'origin', 'uri', 'plink', 'iaid', 'icid', 'tag', 'file'], $condition); + $items = dba::select('item', ['id', 'origin', 'uri', 'created', 'uri-hash', 'iaid', 'icid', 'tag', 'file'], $condition); $content_fields = []; foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) { if (isset($fields[$field])) { $content_fields[$field] = $fields[$field]; - if (in_array($field, self::CONTENT_FIELDLIST)) { + if (in_array($field, self::CONTENT_FIELDLIST) || !self::isLegacyMode()) { unset($fields[$field]); } else { $fields[$field] = null; @@ -759,18 +785,41 @@ class Item extends BaseObject $rows = dba::affected_rows(); while ($item = dba::fetch($items)) { - if (!empty($item['plink'])) { - $content_fields['plink'] = $item['plink']; + // This part here can safely be removed when the legacy fields in the item had been removed + if (empty($item['uri-hash']) && !empty($item['uri']) && !empty($item['created'])) { + + // Fetch the uri-hash from an existing item entry if there is one + $item_condition = ["`uri` = ? AND `uri-hash` != ''", $item['uri']]; + $existing = dba::selectfirst('item', ['uri-hash'], $item_condition); + if (DBM::is_result($existing)) { + $item['uri-hash'] = $existing['uri-hash']; + } else { + $item['uri-hash'] = self::itemHash($item['uri'], $item['created']); + } + + dba::update('item', ['uri-hash' => $item['uri-hash']], ['id' => $item['id']]); + dba::update('item-activity', ['uri-hash' => $item['uri-hash']], ["`uri` = ? AND `uri-hash` = ''", $item['uri']]); + dba::update('item-content', ['uri-plink-hash' => $item['uri-hash']], ["`uri` = ? AND `uri-plink-hash` = ''", $item['uri']]); } + if (!empty($item['iaid']) || (!empty($content_fields['verb']) && (self::activityToIndex($content_fields['verb']) >= 0))) { - self::updateActivity($content_fields, ['uri' => $item['uri']]); + if (!empty($item['iaid'])) { + $update_condition = ['id' => $item['iaid']]; + } else { + $update_condition = ['uri-hash' => $item['uri-hash']]; + } + self::updateActivity($content_fields, $update_condition); if (empty($item['iaid'])) { - $item_activity = dba::selectFirst('item-activity', ['id'], ['uri' => $item['uri']]); + $item_activity = dba::selectFirst('item-activity', ['id'], ['uri-hash' => $item['uri-hash']]); if (DBM::is_result($item_activity)) { $item_fields = ['iaid' => $item_activity['id'], 'icid' => null]; foreach (self::MIXED_CONTENT_FIELDLIST as $field) { - $item_fields[$field] = ''; + if (self::isLegacyMode()) { + $item_fields[$field] = null; + } else { + unset($item_fields[$field]); + } } dba::update('item', $item_fields, ['id' => $item['id']]); @@ -786,16 +835,25 @@ class Item extends BaseObject } } } else { - self::updateContent($content_fields, ['uri' => $item['uri']]); + if (!empty($item['icid'])) { + $update_condition = ['id' => $item['icid']]; + } else { + $update_condition = ['uri-plink-hash' => $item['uri-hash']]; + } + self::updateContent($content_fields, $update_condition); if (empty($item['icid'])) { - $item_content = dba::selectFirst('item-content', [], ['uri' => $item['uri']]); + $item_content = dba::selectFirst('item-content', [], ['uri-plink-hash' => $item['uri-hash']]); if (DBM::is_result($item_content)) { $item_fields = ['icid' => $item_content['id']]; // Clear all fields in the item table that have a content in the item-content table foreach ($item_content as $field => $content) { if (in_array($field, self::MIXED_CONTENT_FIELDLIST) && !empty($item_content[$field])) { - $item_fields[$field] = ''; + if (self::isLegacyMode()) { + $item_fields[$field] = null; + } else { + unset($item_fields[$field]); + } } } dba::update('item', $item_fields, ['id' => $item['id']]); @@ -1228,6 +1286,13 @@ class Item extends BaseObject } } + // Ensure to always have the same creation date. + $existing = dba::selectfirst('item', ['created', 'uri-hash'], ['uri' => $item['uri']]); + if (DBM::is_result($existing)) { + $item['created'] = $existing['created']; + $item['uri-hash'] = $existing['uri-hash']; + } + self::addLanguageToItemArray($item); $item['wall'] = intval(defaults($item, 'wall', 0)); @@ -1272,6 +1337,9 @@ class Item extends BaseObject $item['inform'] = trim(defaults($item, 'inform', '')); $item['file'] = trim(defaults($item, 'file', '')); + // Unique identifier to be linked against item-activities and item-content + $item['uri-hash'] = defaults($item, 'uri-hash', self::itemHash($item['uri'], $item['created'])); + // When there is no content then we don't post it if ($item['body'].$item['title'] == '') { logger('No body, no title.'); @@ -1288,10 +1356,6 @@ class Item extends BaseObject $item['edited'] = DateTimeFormat::utcNow(); } - if (($item['author-link'] == "") && ($item['owner-link'] == "")) { - logger("Both author-link and owner-link are empty. Called by: " . System::callstack(), LOGGER_DEBUG); - } - $item['plink'] = defaults($item, 'plink', System::baseUrl() . '/display/' . urlencode($item['guid'])); // The contact-id should be set before "self::insert" was called - but there seems to be issues sometimes @@ -1718,9 +1782,7 @@ class Item extends BaseObject } $fields = ['uri' => $item['uri'], 'activity' => $activity_index, - 'uri-hash' => hash('sha1', $item['uri']) . hash('ripemd160', $item['uri'])]; - - $saved_item = $item; + 'uri-hash' => $item['uri-hash']]; // We just remove everything that is content foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) { @@ -1734,7 +1796,7 @@ class Item extends BaseObject } // Do we already have this content? - $item_activity = dba::selectFirst('item-activity', ['id'], ['uri' => $item['uri']]); + $item_activity = dba::selectFirst('item-activity', ['id'], ['uri-hash' => $item['uri-hash']]); if (DBM::is_result($item_activity)) { $item['iaid'] = $item_activity['id']; logger('Fetched activity for URI ' . $item['uri'] . ' (' . $item['iaid'] . ')'); @@ -1742,9 +1804,8 @@ class Item extends BaseObject $item['iaid'] = dba::lastInsertId(); logger('Inserted activity for URI ' . $item['uri'] . ' (' . $item['iaid'] . ')'); } else { - // This shouldn't happen. But if it does, we simply store it in the item-content table + // This shouldn't happen. logger('Could not insert activity for URI ' . $item['uri'] . ' - should not happen'); - $item = $saved_item; return false; } if ($locked) { @@ -1760,8 +1821,7 @@ class Item extends BaseObject */ private static function insertContent(&$item) { - $fields = ['uri' => $item['uri'], 'plink' => $item['plink'], - 'uri-plink-hash' => hash('sha1', $item['plink']).hash('sha1', $item['uri'])]; + $fields = ['uri' => $item['uri'], 'uri-plink-hash' => $item['uri-hash']]; foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) { if (isset($item[$field])) { @@ -1777,7 +1837,7 @@ class Item extends BaseObject } // Do we already have this content? - $item_content = dba::selectFirst('item-content', ['id'], ['uri' => $item['uri']]); + $item_content = dba::selectFirst('item-content', ['id'], ['uri-plink-hash' => $item['uri-hash']]); if (DBM::is_result($item_content)) { $item['icid'] = $item_content['id']; logger('Fetched content for URI ' . $item['uri'] . ' (' . $item['icid'] . ')'); @@ -1785,32 +1845,14 @@ class Item extends BaseObject $item['icid'] = dba::lastInsertId(); logger('Inserted content for URI ' . $item['uri'] . ' (' . $item['icid'] . ')'); } else { - // By setting the ICID value through the worker we should avoid timing problems. - // When the locking works, this shouldn't be needed. But better be prepared. - Worker::add(PRIORITY_HIGH, 'SetItemContentID', $item['uri']); - logger('Could not insert content for URI ' . $item['uri'] . ' - trying asynchronously'); + // This shouldn't happen. + logger('Could not insert content for URI ' . $item['uri'] . ' - should not happen'); } if ($locked) { Lock::release('item_insert_content'); } } - /** - * @brief Set the item content id for a given URI - * - * @param string $uri The item URI - */ - public static function setICIDforURI($uri) - { - $item_content = dba::selectFirst('item-content', ['id'], ['uri' => $uri]); - if (DBM::is_result($item_content)) { - dba::update('item', ['icid' => $item_content['id']], ['icid' => 0, 'uri' => $uri]); - logger('Asynchronously set item content id for URI ' . $uri . ' (' . $item_content['id'] . ') - Affected: '. (int)dba::affected_rows()); - } else { - logger('No item-content found for URI ' . $uri); - } - } - /** * @brief Update existing item content entries * @@ -1828,10 +1870,9 @@ class Item extends BaseObject return false; } - $fields = ['activity' => $activity_index, - 'uri-hash' => hash('sha1', $condition['uri']) . hash('ripemd160', $condition['uri'])]; + $fields = ['activity' => $activity_index]; - logger('Update activity for URI ' . $condition['uri']); + logger('Update activity for ' . json_encode($condition)); dba::update('item-activity', $fields, $condition, true); @@ -1860,14 +1901,7 @@ class Item extends BaseObject $fields = $condition; } - if (!empty($item['plink'])) { - $fields['uri-plink-hash'] = hash('sha1', $item['plink']) . hash('sha1', $condition['uri']); - } else { - // Ensure that we don't delete the plink - unset($fields['plink']); - } - - logger('Update content for URI ' . $condition['uri']); + logger('Update content for ' . json_encode($condition)); dba::update('item-content', $fields, $condition, true); } diff --git a/src/Model/ItemContent.php b/src/Model/ItemContent.php index 21dbd34d0a..c17625a756 100644 --- a/src/Model/ItemContent.php +++ b/src/Model/ItemContent.php @@ -93,7 +93,7 @@ class ItemContent extends BaseObject } } - $html = Text\BBCode::convert($post['text'] . $post['after'], false, $htmlmode); + $html = Text\BBCode::convert($post['text'] . defaults($post, 'after', ''), false, $htmlmode); $msg = Text\HTML::toPlaintext($html, 0, true); $msg = trim(html_entity_decode($msg, ENT_QUOTES, 'UTF-8')); @@ -102,7 +102,7 @@ class ItemContent extends BaseObject if ($post['type'] == 'link') { $link = $post['url']; } elseif ($post['type'] == 'text') { - $link = $post['url']; + $link = defaults($post, 'url', ''); } elseif ($post['type'] == 'video') { $link = $post['url']; } elseif ($post['type'] == 'photo') { diff --git a/src/Network/Probe.php b/src/Network/Probe.php index d706089e8d..6b278a965f 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -335,7 +335,7 @@ class Probe } if (x($data, "photo")) { - $data["baseurl"] = Network::getUrlMatch(normalise_link($data["baseurl"]), normalise_link($data["photo"])); + $data["baseurl"] = Network::getUrlMatch(normalise_link(defaults($data, "baseurl", "")), normalise_link($data["photo"])); } else { $data["photo"] = System::baseUrl().'/images/person-175.jpg'; } @@ -1142,7 +1142,7 @@ class Probe } // Older Friendica versions had used the "uid" field differently than newer versions - if ($data["nick"] == $data["guid"]) { + if (!empty($data["nick"]) && !empty($data["guid"]) && ($data["nick"] == $data["guid"])) { unset($data["guid"]); } } @@ -1390,16 +1390,16 @@ class Probe } } - $data["location"] = $xpath->query("//p[contains(@class, 'p-locality')]")->item(0)->nodeValue; + $data["location"] = XML::getFirstNodeValue($xpath, "//p[contains(@class, 'p-locality')]"); if ($data["location"] == '') { - $data["location"] = $xpath->query("//p[contains(@class, 'location')]")->item(0)->nodeValue; + $data["location"] = XML::getFirstNodeValue($xpath, "//p[contains(@class, 'location')]"); } - $data["about"] = $xpath->query("//p[contains(@class, 'p-note')]")->item(0)->nodeValue; + $data["about"] = XML::getFirstNodeValue($xpath, "//p[contains(@class, 'p-note')]"); if ($data["about"] == '') { - $data["about"] = $xpath->query("//p[contains(@class, 'summary')]")->item(0)->nodeValue; + $data["about"] = XML::getFirstNodeValue($xpath, "//p[contains(@class, 'summary')]"); } $avatar = $xpath->query("//img[contains(@class, 'u-photo')]")->item(0); diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index d0fc182a2e..282320ff6d 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -2005,6 +2005,9 @@ class Diaspora $datarray["body"] = $verb; + // Diaspora doesn't provide a date for likes + $datarray["changed"] = $datarray["created"] = $datarray["edited"] = DateTimeFormat::utcNow(); + // like on comments have the comment as parent. So we need to fetch the toplevel parent if ($parent_item["id"] != $parent_item["parent"]) { $toplevel = Item::selectFirst(['origin'], ['id' => $parent_item["parent"]]); diff --git a/src/Protocol/PortableContact.php b/src/Protocol/PortableContact.php index 580f458588..a9e06d19a8 100644 --- a/src/Protocol/PortableContact.php +++ b/src/Protocol/PortableContact.php @@ -19,6 +19,7 @@ use Friendica\Network\Probe; use Friendica\Util\DateTimeFormat; use Friendica\Util\Network; use Friendica\Protocol\Diaspora; +use Friendica\Util\XML; use dba; use DOMDocument; use DOMXPath; @@ -916,7 +917,7 @@ class PortableContact return false; } - $server["site_name"] = $xpath->evaluate("//head/title/text()")->item(0)->nodeValue; + $server["site_name"] = XML::getFirstNodeValue($xpath, '//head/title/text()'); return $server; } @@ -1003,7 +1004,7 @@ class PortableContact // Quit if there is a timeout. // But we want to make sure to only quit if we are mostly sure that this server url fits. if (DBM::is_result($gserver) && ($orig_server_url == $server_url) && - ($serverret['errno'] == CURLE_OPERATION_TIMEDOUT)) { + (!$serverret["success"] && ($serverret['errno'] == CURLE_OPERATION_TIMEDOUT))) { logger("Connection to server ".$server_url." timed out.", LOGGER_DEBUG); dba::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => normalise_link($server_url)]); return false; @@ -1018,7 +1019,7 @@ class PortableContact $serverret = Network::curl($server_url."/.well-known/host-meta", false, $redirects, ['timeout' => 20]); // Quit if there is a timeout - if ($serverret['errno'] == CURLE_OPERATION_TIMEDOUT) { + if (!$serverret["success"] && ($serverret['errno'] == CURLE_OPERATION_TIMEDOUT)) { logger("Connection to server ".$server_url." timed out.", LOGGER_DEBUG); dba::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => normalise_link($server_url)]); return false; @@ -1230,8 +1231,14 @@ class PortableContact $site_name = $data->site->name; $data->site->closed = self::toBoolean($data->site->closed); - $data->site->private = self::toBoolean($data->site->private); - $data->site->inviteonly = self::toBoolean($data->site->inviteonly); + + if (!empty($data->site->private)) { + $data->site->private = self::toBoolean($data->site->private); + } + + if (!empty($data->site->inviteonly)) { + $data->site->inviteonly = self::toBoolean($data->site->inviteonly); + } if (!$data->site->closed && !$data->site->private and $data->site->inviteonly) { $register_policy = REGISTER_APPROVE; @@ -1325,7 +1332,9 @@ class PortableContact $noscrape = $data->no_scrape_url; } $version = $data->version; - $site_name = $data->site_name; + if (!empty($data->site_name)) { + $site_name = $data->site_name; + } $info = $data->info; $register_policy = constant($data->register_policy); $platform = $data->platform; @@ -1714,7 +1723,9 @@ class PortableContact $contact_type = -1; $generation = $default_generation; - $name = $entry->displayName; + if (!empty($entry->displayName)) { + $name = $entry->displayName; + } if (isset($entry->urls)) { foreach ($entry->urls as $url) { diff --git a/src/Worker/CronJobs.php b/src/Worker/CronJobs.php index e843b5920e..80839874da 100644 --- a/src/Worker/CronJobs.php +++ b/src/Worker/CronJobs.php @@ -36,7 +36,8 @@ class CronJobs // Call possible post update functions // see src/Database/PostUpdate.php for more details if ($command == 'post_update') { - PostUpdate::update(); +// Post updates will be reenabled (hopefully in a few days) when most item works are done +// PostUpdate::update(); return; } diff --git a/src/Worker/Delivery.php b/src/Worker/Delivery.php index 8b5b96d5b9..b2781d5874 100644 --- a/src/Worker/Delivery.php +++ b/src/Worker/Delivery.php @@ -44,6 +44,7 @@ class Delivery extends BaseObject return; } $uid = $target_item['uid']; + $items = []; } elseif ($cmd == self::SUGGESTION) { $target_item = dba::selectFirst('fsuggest', [], ['id' => $item_id]); if (!DBM::is_result($target_item)) { @@ -127,6 +128,10 @@ class Delivery extends BaseObject } } + if (empty($items)) { + logger('No delivery data for ' . $cmd . ' - Item ID: ' .$item_id . ' - Contact ID: ' . $contact_id); + } + $owner = User::getOwnerDataById($uid); if (!DBM::is_result($owner)) { return; @@ -271,7 +276,7 @@ class Delivery extends BaseObject // We don't have a relationship with contacts on a public post. // Se we transmit with the new method and via Diaspora as a fallback - if (($items[0]['uid'] == 0) || ($contact['uid'] == 0)) { + if (!empty($items) && (($items[0]['uid'] == 0) || ($contact['uid'] == 0))) { // Transmit in public if it's a relay post $public_dfrn = ($contact['contact-type'] == ACCOUNT_TYPE_RELAY); diff --git a/src/Worker/Expire.php b/src/Worker/Expire.php index b09db5c677..6cc333dd05 100644 --- a/src/Worker/Expire.php +++ b/src/Worker/Expire.php @@ -42,12 +42,12 @@ class Expire { // Normally we shouldn't have orphaned data at all. // If we do have some, then we have to check why. logger('Deleting orphaned item activities - start', LOGGER_DEBUG); - $condition = ["NOT EXISTS (SELECT `iaid` FROM `item` WHERE `item`.`uri` = `item-activity`.`uri`)"]; + $condition = ["NOT EXISTS (SELECT `iaid` FROM `item` WHERE `item`.`iaid` = `item-activity`.`id`)"]; dba::delete('item-activity', $condition); logger('Orphaned item activities deleted: ' . dba::affected_rows(), LOGGER_DEBUG); logger('Deleting orphaned item content - start', LOGGER_DEBUG); - $condition = ["NOT EXISTS (SELECT `icid` FROM `item` WHERE `item`.`uri` = `item-content`.`uri`)"]; + $condition = ["NOT EXISTS (SELECT `icid` FROM `item` WHERE `item`.`icid` = `item-content`.`id`)"]; dba::delete('item-content', $condition); logger('Orphaned item content deleted: ' . dba::affected_rows(), LOGGER_DEBUG); diff --git a/src/Worker/Notifier.php b/src/Worker/Notifier.php index 96549233e5..6c3717200b 100644 --- a/src/Worker/Notifier.php +++ b/src/Worker/Notifier.php @@ -378,7 +378,7 @@ class Notifier { } // If this is a public message and pubmail is set on the parent, include all your email contacts - if (function_exists('imap_open') && !Config::get('system','imap_disabled')) { + if (!empty($target_item) && function_exists('imap_open') && !Config::get('system','imap_disabled')) { if (!strlen($target_item['allow_cid']) && !strlen($target_item['allow_gid']) && !strlen($target_item['deny_cid']) && !strlen($target_item['deny_gid']) && intval($target_item['pubmail'])) { @@ -412,7 +412,7 @@ class Notifier { // delivery loop if (DBM::is_result($r)) { foreach ($r as $contact) { - logger("Deliver ".$target_item["guid"]." to ".$contact['url']." via network ".$contact['network'], LOGGER_DEBUG); + logger("Deliver ".$item_id." to ".$contact['url']." via network ".$contact['network'], LOGGER_DEBUG); Worker::add(['priority' => $a->queue['priority'], 'created' => $a->queue['created'], 'dont_fork' => true], 'Delivery', $cmd, $item_id, (int)$contact['id']); diff --git a/src/Worker/SetItemContentID.php b/src/Worker/SetItemContentID.php deleted file mode 100644 index 96863b3600..0000000000 --- a/src/Worker/SetItemContentID.php +++ /dev/null @@ -1,21 +0,0 @@ -