New language detection

diff --git a/composer.json b/composer.json
index 71b7499dbd53d0c62adc08ed104072b5e0c5da2f..f103ce2fe0ce59b8b9f4e9b3ecd39d12c515c199 100644 (file)
--- a/composer.json
+++ b/composer.json
@@ -40,7 +40,6 @@
                "nikic/fast-route": "^1.3",
                "paragonie/hidden-string": "^1.0",
                "pear/console_table": "^1.3",
-               "pear/text_languagedetect": "1.*",
                "pragmarx/google2fa": "^5.0",
                "pragmarx/recovery": "^0.1.0",
                "psr/container": "^1.0",
@@ -64,7 +63,8 @@
                "npm-asset/perfect-scrollbar": "0.6.16",
                "npm-asset/textcomplete": "^0.18.2",
                "npm-asset/typeahead.js": "^0.11.1",
-               "phpseclib/phpseclib": "^2.0"
+               "phpseclib/phpseclib": "^2.0",
+               "patrickschur/language-detection": "^3.4"
        },
        "repositories": [
                {

diff --git a/composer.lock b/composer.lock
index 26412f83a2d1df9a18266c36d06ccf60a52dcf66..de1a1bdc0b77747961adbeade78d4b46736070d5 100644 (file)
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "ed9aa898eaf8a1f8a807f3be9eecc3d7",
+    "content-hash": "2cd2011a7e93f6f64524d1663651c016",
     "packages": [
         {
             "name": "asika/simple-console",
@@ -460,7 +460,6 @@
                     "jsonld.php"
                 ]
             },
-            "notification-url": "https://packagist.org/downloads/",
             "license": [
                 "BSD-3-Clause"
             ],
@@ -478,11 +477,11 @@
             "description": "A JSON-LD Processor and API implementation in PHP.",
             "homepage": "https://git.friendi.ca/friendica/php-json-ld",
             "keywords": [
+                "JSON",
                 "JSON-LD",
                 "Linked Data",
                 "RDF",
                 "Semantic Web",
-                "json",
                 "jsonld"
             ],
             "time": "2018-10-08T20:41:00+00:00"
@@ -2379,6 +2378,52 @@
             ],
             "time": "2020-03-20T21:48:09+00:00"
         },
+        {
+            "name": "patrickschur/language-detection",
+            "version": "v3.4.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/patrickschur/language-detection.git",
+                "reference": "95b55109177d5c4bd6b1bec6e8835cd0df36ef5f"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/patrickschur/language-detection/zipball/95b55109177d5c4bd6b1bec6e8835cd0df36ef5f",
+                "reference": "95b55109177d5c4bd6b1bec6e8835cd0df36ef5f",
+                "shasum": ""
+            },
+            "require": {
+                "ext-mbstring": "*",
+                "php": "^7"
+            },
+            "require-dev": {
+                "phpunit/phpunit": "^6"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "LanguageDetection\\": "src/LanguageDetection"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Patrick Schur",
+                    "email": "patrick_schur@outlook.de"
+                }
+            ],
+            "description": "A language detection library for PHP. Detects the language from a given text string.",
+            "homepage": "https://github.com/patrickschur/language-detection",
+            "keywords": [
+                "detect",
+                "detection",
+                "language"
+            ],
+            "time": "2018-09-19T21:45:51+00:00"
+        },
         {
             "name": "pear/console_table",
             "version": "v1.3.1",
@@ -2434,50 +2479,6 @@
             ],
             "time": "2018-01-25T20:47:17+00:00"
         },
-        {
-            "name": "pear/text_languagedetect",
-            "version": "v1.0.1",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/pear/Text_LanguageDetect.git",
-                "reference": "9e253f26cef9a9066f53f200cc3e0684018cb5b5"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/pear/Text_LanguageDetect/zipball/9e253f26cef9a9066f53f200cc3e0684018cb5b5",
-                "reference": "9e253f26cef9a9066f53f200cc3e0684018cb5b5",
-                "shasum": ""
-            },
-            "require-dev": {
-                "phpunit/phpunit": "8.*|9.*"
-            },
-            "suggest": {
-                "ext-mbstring": "May require the mbstring PHP extension"
-            },
-            "type": "library",
-            "autoload": {
-                "psr-0": {
-                    "Text": "./"
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "include-path": [
-                "./"
-            ],
-            "license": [
-                "BSD-2-Clause"
-            ],
-            "authors": [
-                {
-                    "name": "Nicholas Pisarro",
-                    "email": "taak@php.net",
-                    "role": "Lead"
-                }
-            ],
-            "description": "Identify human languages from text samples",
-            "homepage": "http://pear.php.net/package/Text_LanguageDetect",
-            "time": "2020-05-17T12:19:40+00:00"
-        },
         {
             "name": "phpseclib/phpseclib",
             "version": "2.0.29",
@@ -4450,7 +4451,7 @@
                 }
             ],
             "description": "Provides the functionality to compare PHP values for equality",
-            "homepage": "https://github.com/sebastianbergmann/comparator",
+            "homepage": "http://www.github.com/sebastianbergmann/comparator",
             "keywords": [
                 "comparator",
                 "compare",
@@ -4552,7 +4553,7 @@
                 }
             ],
             "description": "Provides functionality to handle HHVM/PHP environments",
-            "homepage": "https://github.com/sebastianbergmann/environment",
+            "homepage": "http://www.github.com/sebastianbergmann/environment",
             "keywords": [
                 "Xdebug",
                 "environment",
@@ -4620,7 +4621,7 @@
                 }
             ],
             "description": "Provides the functionality to export PHP variables for visualization",
-            "homepage": "https://github.com/sebastianbergmann/exporter",
+            "homepage": "http://www.github.com/sebastianbergmann/exporter",
             "keywords": [
                 "export",
                 "exporter"
@@ -4672,7 +4673,7 @@
                 }
             ],
             "description": "Snapshotting of global state",
-            "homepage": "https://github.com/sebastianbergmann/global-state",
+            "homepage": "http://www.github.com/sebastianbergmann/global-state",
             "keywords": [
                 "global state"
             ],
@@ -4774,7 +4775,7 @@
                 }
             ],
             "description": "Provides functionality to recursively process PHP variables",
-            "homepage": "https://github.com/sebastianbergmann/recursion-context",
+            "homepage": "http://www.github.com/sebastianbergmann/recursion-context",
             "time": "2016-11-19T07:33:16+00:00"
         },
         {

diff --git a/src/Model/Item.php b/src/Model/Item.php
index c384eb4f9484c92c9576b471f983b6856e0f9989..e8ebe5671c0dedb074cadf6c1b67e1f6d963c1b2 100644 (file)
--- a/src/Model/Item.php
+++ b/src/Model/Item.php
@@ -43,8 +43,8 @@ use Friendica\Util\Map;
 use Friendica\Util\Network;
 use Friendica\Util\Strings;
 use Friendica\Worker\Delivery;
-use Text_LanguageDetect;
 use Friendica\Repository\PermissionSet as RepPermissionSet;
+use LanguageDetection\Language;
 
 class Item
 {
@@ -1699,10 +1699,10 @@ class Item
 
                $item['plink'] = ($item['plink'] ?? '') ?: DI::baseUrl() . '/display/' . urlencode($item['guid']);
 
-               $item['language'] = self::getLanguage($item);
-
                $item['gravity'] = self::getGravity($item);
 
+               $item['language'] = self::getLanguage($item);
+
                $default = ['url' => $item['author-link'], 'name' => $item['author-name'],
                        'photo' => $item['author-avatar'], 'network' => $item['network']];
                $item['author-id'] = ($item['author-id'] ?? 0) ?: Contact::getIdForURL($item['author-link'], 0, null, $default);
@@ -2472,11 +2472,14 @@ class Item
         */
        private static function getLanguage(array $item)
        {
+               if (!in_array($item['gravity'], [GRAVITY_PARENT, GRAVITY_COMMENT])) {
+                       return '';
+               }
+
                $naked_body = BBCode::toPlaintext($item['body'], false);
 
-               $ld = new Text_LanguageDetect();
-               $ld->setNameMode(2);
-               $languages = $ld->detect($naked_body, 3);
+               $ld = new Language;
+               $languages = $ld->detect($naked_body)->limit(0, 3)->close();
                if (is_array($languages)) {
                        return json_encode($languages);
                }