The detected language now moved to "item-content" as well

diff --git a/boot.php b/boot.php
index 864e7804c917cedca82c72dd51ae75844ad536a8..786a846f320424f12d7cbbd8d644d7cb6f152668 100644 (file)
--- a/boot.php
+++ b/boot.php
@@ -41,7 +41,7 @@ define('FRIENDICA_PLATFORM',     'Friendica');
 define('FRIENDICA_CODENAME',     'The Tazmans Flax-lily');
 define('FRIENDICA_VERSION',      '2018.08-dev');
 define('DFRN_PROTOCOL_VERSION',  '2.23');
-define('DB_UPDATE_VERSION',      1272);
+define('DB_UPDATE_VERSION',      1273);
 define('NEW_UPDATE_ROUTINE_VERSION', 1170);
 
 /**

diff --git a/database.sql b/database.sql
index 1a08a5ba980f6cfe7af8adfb67fbc7b0c15ca120..c452832acde34792a684fefd7a86be2b3fbe67f8 100644 (file)
--- a/database.sql
+++ b/database.sql
@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2018.08-dev (The Tazmans Flax-lily)
--- DB_UPDATE_VERSION 1272
+-- DB_UPDATE_VERSION 1273
 -- ------------------------------------------
 
 
@@ -557,6 +557,7 @@ CREATE TABLE IF NOT EXISTS `item-content` (
        `body` mediumtext COMMENT 'item body content',
        `location` varchar(255) NOT NULL DEFAULT '' COMMENT 'text location where this item originated',
        `coord` varchar(255) NOT NULL DEFAULT '' COMMENT 'longitude/latitude pair representing location where this item originated',
+       `language` text COMMENT 'Language information about this post',
        `app` varchar(255) NOT NULL DEFAULT '' COMMENT 'application which generated this item',
        `rendered-hash` varchar(32) NOT NULL DEFAULT '' COMMENT '',
        `rendered-html` mediumtext COMMENT 'item.body converted to html',

diff --git a/src/Database/DBStructure.php b/src/Database/DBStructure.php
index 8fb2afddb22214cdcac440a7683ad135972f718b..fa49c29032f3e68ff2287cb94c87750c622c853d 100644 (file)
--- a/src/Database/DBStructure.php
+++ b/src/Database/DBStructure.php
@@ -1262,6 +1262,7 @@ class DBStructure
                                                "body" => ["type" => "mediumtext", "comment" => "item body content"],
                                                "location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "text location where this item originated"],
                                                "coord" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "longitude/latitude pair representing location where this item originated"],
+                                               "language" => ["type" => "text", "comment" => "Language information about this post"],
                                                "app" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "application which generated this item"],
                                                "rendered-hash" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => ""],
                                                "rendered-html" => ["type" => "mediumtext", "comment" => "item.body converted to html"],

diff --git a/src/Model/Item.php b/src/Model/Item.php
index 12046d4ffca40d634771c8cc5d84151d2e2575ee..5fe03a81b6d661fe792a3d633cf4f6cb96260a13 100644 (file)
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -37,7 +37,7 @@ class Item extends BaseObject
        // Field list that is used to display the items
        const DISPLAY_FIELDLIST = ['uid', 'id', 'parent', 'uri', 'thr-parent', 'parent-uri', 'guid', 'network',
                        'commented', 'created', 'edited', 'received', 'verb', 'object-type', 'postopts', 'plink',
-                       'wall', 'private', 'starred', 'origin', 'title', 'body', 'file', 'attach',
+                       'wall', 'private', 'starred', 'origin', 'title', 'body', 'file', 'attach', 'language',
                        'content-warning', 'location', 'coord', 'app', 'rendered-hash', 'rendered-html', 'object',
                        'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'item_id',
                        'author-id', 'author-link', 'author-name', 'author-avatar',
@@ -58,10 +58,13 @@ class Item extends BaseObject
                        'signed_text', 'signature', 'signer'];
 
        // Field list for "item-content" table that is mixed with the item table
-       const CONTENT_FIELDLIST = ['title', 'content-warning', 'body', 'location',
+       const MIXED_CONTENT_FIELDLIST = ['title', 'content-warning', 'body', 'location',
                        'coord', 'app', 'rendered-hash', 'rendered-html', 'verb',
                        'object-type', 'object', 'target-type', 'target', 'plink'];
 
+       // Field list for "item-content" table that is not present in the "item" table
+       const CONTENT_FIELDLIST = ['language'];
+
        // All fields in the item table
        const ITEM_FIELDLIST = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent', 'guid',
                        'contact-id', 'type', 'wall', 'gravity', 'extid', 'icid',
@@ -86,7 +89,7 @@ class Item extends BaseObject
                $row = dba::fetch($stmt);
 
                // Fetch data from the item-content table whenever there is content there
-               foreach (self::CONTENT_FIELDLIST as $field) {
+               foreach (self::MIXED_CONTENT_FIELDLIST as $field) {
                        if (empty($row[$field]) && !empty($row['item-' . $field])) {
                                $row[$field] = $row['item-' . $field];
                        }
@@ -406,7 +409,7 @@ class Item extends BaseObject
                        'unseen', 'deleted', 'origin', 'forum_mode', 'mention', 'global',
                        'id' => 'item_id', 'network', 'icid'];
 
-               $fields['item-content'] = self::CONTENT_FIELDLIST;
+               $fields['item-content'] = array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST);
 
                $fields['author'] = ['url' => 'author-link', 'name' => 'author-name',
                        'thumb' => 'author-avatar', 'nick' => 'author-nick'];
@@ -526,7 +529,7 @@ class Item extends BaseObject
                foreach ($fields as $table => $table_fields) {
                        foreach ($table_fields as $field => $select) {
                                if (empty($selected) || in_array($select, $selected)) {
-                                       if (in_array($select, self::CONTENT_FIELDLIST)) {
+                                       if (in_array($select, self::MIXED_CONTENT_FIELDLIST)) {
                                                $selection[] = "`item`.`".$select."` AS `item-" . $select . "`";
                                        }
                                        if (is_int($field)) {
@@ -594,7 +597,7 @@ class Item extends BaseObject
                $items = dba::select('item', ['id', 'origin', 'uri', 'plink'], $condition);
 
                $content_fields = [];
-               foreach (self::CONTENT_FIELDLIST as $field) {
+               foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) {
                        if (isset($fields[$field])) {
                                $content_fields[$field] = $fields[$field];
                                unset($fields[$field]);
@@ -1032,7 +1035,7 @@ class Item extends BaseObject
                        }
                }
 
-               self::addLanguageInPostopts($item);
+               self::addLanguageToItemArray($item);
 
                $item['wall']          = intval(defaults($item, 'wall', 0));
                $item['extid']         = trim(defaults($item, 'extid', ''));
@@ -1498,7 +1501,7 @@ class Item extends BaseObject
                $fields = ['uri' => $item['uri'], 'plink' => $item['plink'],
                        'uri-plink-hash' => hash('sha1', $item['plink']).hash('sha1', $item['uri'])];
 
-               foreach (self::CONTENT_FIELDLIST as $field) {
+               foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) {
                        if (isset($item[$field])) {
                                $fields[$field] = $item[$field];
                                unset($item[$field]);
@@ -1556,7 +1559,7 @@ class Item extends BaseObject
        {
                // We have to select only the fields from the "item-content" table
                $fields = [];
-               foreach (self::CONTENT_FIELDLIST as $field) {
+               foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) {
                        if (isset($item[$field])) {
                                $fields[$field] = $item[$field];
                        }
@@ -1822,39 +1825,19 @@ class Item extends BaseObject
        }
 
         /**
-        * Adds a "lang" specification in a "postopts" element of given $arr,
-        * if possible and not already present.
+        * Adds a language specification in a "language" element of given $arr.
         * Expects "body" element to exist in $arr.
         */
-       private static function addLanguageInPostopts(&$item)
+       private static function addLanguageToItemArray(&$item)
        {
-               $postopts = "";
-
-               if (!empty($item['postopts'])) {
-                       if (strstr($item['postopts'], 'lang=')) {
-                               // do not override
-                               return;
-                       }
-                       $postopts = $item['postopts'];
-               }
-
                $naked_body = Text\BBCode::toPlaintext($item['body'], false);
 
-               $languages = (new Text_LanguageDetect())->detect($naked_body, 3);
-
-               if (sizeof($languages) > 0) {
-                       if ($postopts != '') {
-                               $postopts .= '&'; // arbitrary separator, to be reviewed
-                       }
+               $ld = new Text_LanguageDetect();
+               $ld->setNameMode(2);
+               $languages = $ld->detect($naked_body, 3);
 
-                       $postopts .= 'lang=';
-                       $sep = "";
-
-                       foreach ($languages as $language => $score) {
-                               $postopts .= $sep . $language . ";" . $score;
-                               $sep = ':';
-                       }
-                       $item['postopts'] = $postopts;
+               if (is_array($languages)) {
+                       $item['language'] = json_encode($languages);
                }
        }