We now store the uri data in a separate table (#5560)

* We now have a item-uri table

* Fixing line endings

* New item uri table

* Rescued ItemURI.php file

* Reverting some git problem

* Corrected the dbstructure file

* Updated database.sql / we now store content in the new id fields

diff --git a/boot.php b/boot.php
index f660b65b5163a23bac5ca22050ab5085eb43a081..b8149684c3bd8ae202be56dd7fb3fab416614475 100644 (file)
--- a/boot.php
+++ b/boot.php
@@ -41,7 +41,7 @@ define('FRIENDICA_PLATFORM',     'Friendica');
 define('FRIENDICA_CODENAME',     'The Tazmans Flax-lily');
 define('FRIENDICA_VERSION',      '2018.08-dev');
 define('DFRN_PROTOCOL_VERSION',  '2.23');
-define('DB_UPDATE_VERSION',      1280);
+define('DB_UPDATE_VERSION',      1281);
 define('NEW_UPDATE_ROUTINE_VERSION', 1170);
 
 /**

diff --git a/config/dbstructure.json b/config/dbstructure.json
index fa7513332718c20012fd3c1b7af8e02ff1498804..fce2fcf930d72619783505ad7874e0feeaa46819 100644 (file)
--- a/config/dbstructure.json
+++ b/config/dbstructure.json
@@ -211,7 +211,7 @@
        "conversation": {
                "comment": "Raw data and structure information for messages",
                "fields": {
-                       "item-uri": {"type": "varbinary(255)", "not null": "1", "primary": "1", "comment": "URI of the item"},
+                       "item-uri": {"type": "varbinary(255)", "not null": "1", "primary": "1", "comment": "Original URI of the item - unrelated to the table with the same name"},
                        "reply-to-uri": {"type": "varbinary(255)", "not null": "1", "default": "", "comment": "URI to which this item is a reply"},
                        "conversation-uri": {"type": "varbinary(255)", "not null": "1", "default": "", "comment": "GNU Social conversation URI"},
                        "conversation-href": {"type": "varbinary(255)", "not null": "1", "default": "", "comment": "GNU Social conversation link"},
@@ -471,10 +471,13 @@
                        "id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1", "relation": {"thread": "iid"}},
                        "guid": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "A unique identifier for this item"},
                        "uri": {"type": "varchar(255)", "not null": "1", "default": "", "comment": ""},
+                       "uri-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table entry that contains the item uri"},
                        "uri-hash": {"type": "varchar(80)", "not null": "1", "default": "", "comment": "RIPEMD-128 hash from uri"},
                        "parent": {"type": "int unsigned", "not null": "1", "default": "0", "relation": {"item": "id"}, "comment": "item.id of the parent to this item if it is a reply of some form; otherwise this must be set to the id of this item"},
                        "parent-uri": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "uri of the parent to this item"},
+                       "parent-uri-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table that contains the parent uri"},
                        "thr-parent": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "If the parent of this item is not the top-level item in the conversation, the uri of the immediate parent; otherwise set to parent-uri"},
+                       "thr-parent-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table that contains the thread parent uri"},
                        "created": {"type": "datetime", "not null": "1", "default": "0001-01-01 00:00:00", "comment": "Creation timestamp."},
                        "edited": {"type": "datetime", "not null": "1", "default": "0001-01-01 00:00:00", "comment": "Date of last edit (default is created)"},
                        "commented": {"type": "datetime", "not null": "1", "default": "0001-01-01 00:00:00", "comment": "Date of last comment/reply to this item"},
@@ -571,6 +574,7 @@
                "fields": {
                        "id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1", "relation": {"thread": "iid"}},
                        "uri": {"type": "varchar(255)", "comment": ""},
+                       "uri-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table entry that contains the item uri"},
                        "uri-hash": {"type": "varchar(80)", "not null": "1", "default": "", "comment": "RIPEMD-128 hash from uri"},
                        "activity": {"type": "smallint unsigned", "not null": "1", "default": "0", "comment": ""}
                },
@@ -585,6 +589,7 @@
                "fields": {
                        "id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1", "relation": {"thread": "iid"}},
                        "uri": {"type": "varchar(255)", "comment": ""},
+                       "uri-id": {"type": "int unsigned", "relation": {"item-uri": "id"}, "comment": "Id of the item-uri table entry that contains the item uri"},
                        "uri-plink-hash": {"type": "varchar(80)", "not null": "1", "default": "", "comment": "RIPEMD-128 hash from uri"},
                        "title": {"type": "varchar(255)", "not null": "1", "default": "", "comment": "item title"},
                        "content-warning": {"type": "varchar(255)", "not null": "1", "default": "", "comment": ""},
@@ -619,6 +624,19 @@
                        "PRIMARY": ["iid"]
                }
        },
+       "item-uri": {
+               "comment": "URI and GUID for items",
+               "fields": {
+                       "id": {"type": "int unsigned", "not null": "1", "extra": "auto_increment", "primary": "1"},
+                       "uri": {"type": "varbinary(255)", "not null": "1", "comment": "URI of an item"},
+                       "guid": {"type": "varbinary(255)", "comment": "A unique identifier for an item"}
+               },
+               "indexes": {
+                       "PRIMARY": ["id"],
+                       "uri": ["UNIQUE", "uri"],
+                       "guid": ["guid"]
+               }
+       },
        "locks": {
                "comment": "",
                "fields": {
@@ -1247,4 +1265,4 @@
                        "done_executed": ["done", "executed"]
                }
        }
-}
\ No newline at end of file
+}

diff --git a/database.sql b/database.sql
index 6dc3b407bfea782b9130efbb48f55168aa347a00..528932846297541c0bf83d90dd06d34608a88a40 100644 (file)
--- a/database.sql
+++ b/database.sql
@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2018.08-dev (The Tazmans Flax-lily)
--- DB_UPDATE_VERSION 1280
+-- DB_UPDATE_VERSION 1281
 -- ------------------------------------------
 
 
@@ -208,7 +208,7 @@ CREATE TABLE IF NOT EXISTS `conv` (
 -- TABLE conversation
 --
 CREATE TABLE IF NOT EXISTS `conversation` (
-       `item-uri` varbinary(255) NOT NULL COMMENT 'URI of the item',
+       `item-uri` varbinary(255) NOT NULL COMMENT 'Original URI of the item - unrelated to the table with the same name',
        `reply-to-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'URI to which this item is a reply',
        `conversation-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation URI',
        `conversation-href` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation link',
@@ -455,10 +455,13 @@ CREATE TABLE IF NOT EXISTS `item` (
        `id` int unsigned NOT NULL auto_increment,
        `guid` varchar(255) NOT NULL DEFAULT '' COMMENT 'A unique identifier for this item',
        `uri` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+       `uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the item uri',
        `uri-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri',
        `parent` int unsigned NOT NULL DEFAULT 0 COMMENT 'item.id of the parent to this item if it is a reply of some form; otherwise this must be set to the id of this item',
        `parent-uri` varchar(255) NOT NULL DEFAULT '' COMMENT 'uri of the parent to this item',
+       `parent-uri-id` int unsigned COMMENT 'Id of the item-uri table that contains the parent uri',
        `thr-parent` varchar(255) NOT NULL DEFAULT '' COMMENT 'If the parent of this item is not the top-level item in the conversation, the uri of the immediate parent; otherwise set to parent-uri',
+       `thr-parent-id` int unsigned COMMENT 'Id of the item-uri table that contains the thread parent uri',
        `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Creation timestamp.',
        `edited` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last edit (default is created)',
        `commented` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last comment/reply to this item',
@@ -554,6 +557,7 @@ CREATE TABLE IF NOT EXISTS `item` (
 CREATE TABLE IF NOT EXISTS `item-activity` (
        `id` int unsigned NOT NULL auto_increment,
        `uri` varchar(255) COMMENT '',
+       `uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the item uri',
        `uri-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri',
        `activity` smallint unsigned NOT NULL DEFAULT 0 COMMENT '',
         PRIMARY KEY(`id`),
@@ -567,6 +571,7 @@ CREATE TABLE IF NOT EXISTS `item-activity` (
 CREATE TABLE IF NOT EXISTS `item-content` (
        `id` int unsigned NOT NULL auto_increment,
        `uri` varchar(255) COMMENT '',
+       `uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the item uri',
        `uri-plink-hash` varchar(80) NOT NULL DEFAULT '' COMMENT 'RIPEMD-128 hash from uri',
        `title` varchar(255) NOT NULL DEFAULT '' COMMENT 'item title',
        `content-warning` varchar(255) NOT NULL DEFAULT '' COMMENT '',
@@ -598,6 +603,18 @@ CREATE TABLE IF NOT EXISTS `item-delivery-data` (
         PRIMARY KEY(`iid`)
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Delivery data for items';
 
+--
+-- TABLE item-uri
+--
+CREATE TABLE IF NOT EXISTS `item-uri` (
+       `id` int unsigned NOT NULL auto_increment,
+       `uri` varbinary(255) NOT NULL COMMENT 'URI of an item',
+       `guid` varbinary(255) COMMENT 'A unique identifier for an item',
+        PRIMARY KEY(`id`),
+        UNIQUE INDEX `uri` (`uri`),
+        INDEX `guid` (`guid`)
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='URI and GUID for items';
+
 --
 -- TABLE locks
 --

diff --git a/src/Model/Item.php b/src/Model/Item.php
index 3fe69fb12793dbce4a62710d6ccd0ec5ec1ba3d1..84c51a2bf0d4e8bdce3f9f2e68b892181703617b 100644 (file)
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -17,6 +17,7 @@ use Friendica\Core\Worker;
 use Friendica\Database\DBA;
 use Friendica\Model\Contact;
 use Friendica\Model\PermissionSet;
+use Friendica\Model\ItemURI;
 use Friendica\Object\Image;
 use Friendica\Protocol\Diaspora;
 use Friendica\Protocol\OStatus;
@@ -1253,6 +1254,9 @@ class Item extends BaseObject
                $item['guid'] = self::guid($item, $notify);
                $item['uri'] = notags(trim(defaults($item, 'uri', self::newURI($item['uid'], $item['guid']))));
 
+               // Store URI data
+               $item['uri-id'] = ItemURI::insert(['uri' => $item['uri'], 'guid' => $item['guid']]);
+
                // Store conversation data
                $item = Conversation::insert($item);
 
@@ -1565,6 +1569,9 @@ class Item extends BaseObject
                        }
                }
 
+               $item['parent-uri-id'] = ItemURI::getIdByURI($item['parent-uri']);
+               $item['thr-parent-id'] = ItemURI::getIdByURI($item['thr-parent']);
+
                $condition = ["`uri` = ? AND `network` IN (?, ?) AND `uid` = ?",
                        $item['uri'], $item['network'], NETWORK_DFRN, $item['uid']];
                if (self::exists($condition)) {

diff --git a/src/Model/ItemURI.php b/src/Model/ItemURI.php
new file mode 100644 (file)
index 0000000..559babb
--- /dev/null
+++ b/src/Model/ItemURI.php
@@ -0,0 +1,56 @@
+<?php
+
+/**
+ * @file src/Model/ItemURI.php
+ */
+
+namespace Friendica\Model;
+
+use Friendica\BaseObject;
+use Friendica\Database\DBA;
+
+require_once 'boot.php';
+
+class ItemURI extends BaseObject
+{
+       /**
+        * @brief Insert an item-uri record and return its id
+        *
+        * @param array $fields Item-uri fields
+        *
+        * @return integer item-uri id
+        */
+       public static function insert($fields)
+       {
+               if (!DBA::exists('item-uri', ['uri' => $fields['uri']])) {
+                       DBA::insert('item-uri', $fields, true);
+               }
+
+               $itemuri = DBA::selectFirst('item-uri', ['id'], ['uri' => $fields['uri']]);
+
+               if (!DBA::isResult($itemuri)) {
+                       // This shouldn't happen
+                       return null;
+               }
+
+               return $itemuri['id'];
+       }
+
+       /**
+        * @brief Searched for an id of a given uri. Adds it, if not existing yet.
+        *
+        * @param string $uri
+        *
+        * @return integer item-uri id
+        */
+       public static function getIdByURI($uri)
+       {
+               $itemuri = DBA::selectFirst('item-uri', ['id'], ['uri' => $uri]);
+
+               if (!DBA::isResult($itemuri)) {
+                       return self::insert(['uri' => $uri]);
+               }
+
+               return $itemuri['id'];
+       }
+}