X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=src%2FModel%2FItem.php;h=18b56215ce39ca9a680e64fc062f693c84dca392;hb=58e5f0d9c5ba75f92cc4c77b5b27069789f11318;hp=27bb58c78e9b9b3fd1ee7cb507afaa7dd0b323e1;hpb=8968c63fcc49a2e75eee003457f7852a9163a4ca;p=friendica.git diff --git a/src/Model/Item.php b/src/Model/Item.php index 27bb58c78e..18b56215ce 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -117,7 +117,7 @@ class Item const DELIVER_FIELDLIST = [ 'uid', 'id', 'parent', 'uri-id', 'uri', 'thr-parent', 'parent-uri', 'guid', 'parent-guid', 'conversation', 'received', 'created', 'edited', 'verb', 'object-type', 'object', 'target', - 'private', 'title', 'body', 'raw-body', 'location', 'coord', 'app', + 'private', 'title', 'body', 'raw-body', 'language', 'location', 'coord', 'app', 'inform', 'deleted', 'extid', 'post-type', 'post-reason', 'gravity', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'author-id', 'author-addr', 'author-link', 'author-name', 'author-avatar', 'owner-id', 'owner-link', 'contact-uid', @@ -330,7 +330,7 @@ class Item */ public static function markForDeletionById(int $item_id, int $priority = Worker::PRIORITY_HIGH): bool { - Logger::info('Mark item for deletion by id', ['id' => $item_id, 'callstack' => System::callstack()]); + Logger::info('Mark item for deletion by id', ['id' => $item_id]); // locate item to be deleted $fields = [ 'id', 'uri', 'uri-id', 'uid', 'parent', 'parent-uri-id', 'origin', @@ -795,7 +795,7 @@ class Item } if (!DBA::isResult($parent)) { - Logger::notice('item parent was not found - ignoring item', ['uri-id' => $item['uri-id'], 'thr-parent-id' => $item['thr-parent-id'], 'uid' => $item['uid'], 'callstack' => System::callstack(20)]); + Logger::notice('item parent was not found - ignoring item', ['uri-id' => $item['uri-id'], 'thr-parent-id' => $item['thr-parent-id'], 'uid' => $item['uid']]); return []; } @@ -880,6 +880,10 @@ class Item if (is_int($notify) && in_array($notify, Worker::PRIORITIES)) { $priority = $notify; } + + // Mastodon style API visibility + $copy_permissions = ($item['visibility'] ?? 'private') == 'private'; + unset($item['visibility']); } else { $item['network'] = trim(($item['network'] ?? '') ?: Protocol::PHANTOM); } @@ -1038,10 +1042,12 @@ class Item // Reshares have to keep their permissions to allow groups to work if (!$defined_permissions && (!$item['origin'] || ($item['verb'] != Activity::ANNOUNCE))) { - $item['allow_cid'] = $toplevel_parent['allow_cid']; - $item['allow_gid'] = $toplevel_parent['allow_gid']; - $item['deny_cid'] = $toplevel_parent['deny_cid']; - $item['deny_gid'] = $toplevel_parent['deny_gid']; + // Don't store the permissions on pure AP posts + $store_permissions = ($item['network'] != Protocol::ACTIVITYPUB) || $item['origin'] || !empty($item['diaspora_signed_text']); + $item['allow_cid'] = $store_permissions ? $toplevel_parent['allow_cid'] : ''; + $item['allow_gid'] = $store_permissions ? $toplevel_parent['allow_gid'] : ''; + $item['deny_cid'] = $store_permissions ? $toplevel_parent['deny_cid'] : ''; + $item['deny_gid'] = $store_permissions ? $toplevel_parent['deny_gid'] : ''; } $parent_origin = $toplevel_parent['origin']; @@ -1183,7 +1189,7 @@ class Item if (!empty($quote_id)) { // This is one of these "should not happen" situations. // The protocol implementations should already have done this job. - Logger::notice('Quote-uri-id detected in post', ['id' => $quote_id, 'guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'callstack' => System::callstack(20)]); + Logger::notice('Quote-uri-id detected in post', ['id' => $quote_id, 'guid' => $item['guid'], 'uri-id' => $item['uri-id']]); $item['quote-uri-id'] = $quote_id; } } @@ -1212,8 +1218,6 @@ class Item // Check for hashtags in the body and repair or add hashtag links $item['body'] = self::setHashtags($item['body']); - $item['language'] = self::getLanguage($item); - $notify_type = Delivery::POST; // Filling item related side tables @@ -1262,6 +1266,8 @@ class Item } } + $item['language'] = self::getLanguage($item); + $inserted = Post::insert($item['uri-id'], $item); if ($item['gravity'] == self::GRAVITY_PARENT) { @@ -1359,6 +1365,9 @@ class Item if ($notify) { DI::contentItem()->postProcessPost($posted_item); + if ($copy_permissions && ($posted_item['thr-parent-id'] != $posted_item['uri-id']) && ($posted_item['private'] == self::PRIVATE)) { + DI::contentItem()->copyPermissions($posted_item['thr-parent-id'], $posted_item['uri-id']); + } } else { Hook::callAll('post_remote_end', $posted_item); } @@ -1475,6 +1484,10 @@ class Item */ private static function setOwnerforResharedItem(array $item) { + if ($item['uid'] == 0) { + return; + } + $parent = Post::selectFirst( ['id', 'causer-id', 'owner-id', 'author-id', 'author-link', 'origin', 'post-reason'], ['uri-id' => $item['thr-parent-id'], 'uid' => $item['uid']] @@ -1991,7 +2004,7 @@ class Item return ''; } - $languages = self::getLanguageArray(trim($item['title'] . "\n" . $item['body']), 3, $item['uri-id'], $item['author-id']); + $languages = self::getLanguageArray($item['title'] . ' ' . ($item['content-warning'] ?? '') . ' ' . $item['body'], 3, $item['uri-id'], $item['author-id']); if (empty($languages)) { return ''; } @@ -2010,67 +2023,141 @@ class Item */ public static function getLanguageArray(string $body, int $count, int $uri_id = 0, int $author_id = 0): array { - $naked_body = BBCode::toSearchText($body, $uri_id); + $searchtext = BBCode::toSearchText($body, $uri_id); - if ((count(explode(' ', $naked_body)) < 10) && (mb_strlen($naked_body) < 30) && $author_id) { + if ((count(explode(' ', $searchtext)) < 10) && (mb_strlen($searchtext) < 30) && $author_id) { $author = Contact::selectFirst(['about'], ['id' => $author_id]); if (!empty($author['about'])) { $about = BBCode::toSearchText($author['about'], 0); - $about = self::getDominantLanguage($about); - Logger::debug('About field added', ['author' => $author_id, 'body' => $naked_body, 'about' => $about]); - $naked_body .= ' ' . $about; + Logger::debug('About field added', ['author' => $author_id, 'body' => $searchtext, 'about' => $about]); + $searchtext .= ' ' . $about; } } - if (empty($naked_body)) { + if (empty($searchtext)) { return []; } - $naked_body = self::getDominantLanguage($naked_body); + $ld = new Language(DI::l10n()->getDetectableLanguages()); - $availableLanguages = DI::l10n()->getAvailableLanguages(true); - $availableLanguages = DI::l10n()->convertForLanguageDetection($availableLanguages); + $result = []; - $ld = new Language(array_keys($availableLanguages)); - $languages = $ld->detect($naked_body)->limit(0, $count)->close() ?: []; + foreach (self::splitByBlocks($searchtext) as $block) { + $languages = $ld->detect($block)->close() ?: []; - $data = [ - 'text' => $naked_body, - 'detected' => $languages, - 'uri-id' => $uri_id, - ]; + $data = [ + 'text' => $block, + 'detected' => $languages, + 'uri-id' => $uri_id, + 'author-id' => $author_id, + ]; + Hook::callAll('detect_languages', $data); + + foreach ($data['detected'] as $language => $quality) { + $result[$language] = max($result[$language] ?? 0, $quality * (strlen($block) / strlen($searchtext))); + } + } + + $result = self::compactLanguages($result); - Hook::callAll('detect_languages', $data); - $languages = $data['detected']; + arsort($result); + return array_slice($result, 0, $count); + } + /** + * Concert the language code in the detection result to ISO 639-1. + * On duplicates the system uses the higher quality value. + * + * @param array $result + * @return array + */ + private static function compactLanguages(array $result): array + { + $languages = []; + foreach ($result as $language => $quality) { + if ($quality == 0) { + continue; + } + $code = DI::l10n()->toISO6391($language); + if (empty($languages[$code]) || ($languages[$code] < $quality)) { + $languages[$code] = $quality; + } + } return $languages; } /** - * Check if latin or non latin are dominant in the body and only return the dominant one + * Split a string into different unicode blocks + * Currently the text is split into the latin and the non latin part. * * @param string $body - * @return string + * @return array */ - private static function getDominantLanguage(string $body): string + private static function splitByBlocks(string $body): array { - $latin = ''; - $non_latin = ''; + if (!class_exists('IntlChar')) { + return [$body]; + } + + $blocks = []; + $previous_block = 0; + for ($i = 0; $i < mb_strlen($body); $i++) { $character = mb_substr($body, $i, 1); - $ord = mb_ord($character); - - // We add the most common characters to both strings. - if (($ord <= 64) || ($ord >= 91 && $ord <= 96) || ($ord >= 123 && $ord <= 191) || in_array($ord, [215, 247]) || ($ord >= 697 && $ord <= 735) || ($ord > 65535)) { - $latin .= $character; - $non_latin .= $character; - } elseif ($ord < 768) { - $latin .= $character; + $previous = ($i > 0) ? mb_substr($body, $i - 1, 1) : ''; + $next = ($i < mb_strlen($body)) ? mb_substr($body, $i + 1, 1) : ''; + + if (!\IntlChar::isalpha($character)) { + if (($previous != '') && (\IntlChar::isalpha($previous))) { + $previous_block = self::getBlockCode($previous); + } + + $block = (($next != '') && \IntlChar::isalpha($next)) ? self::getBlockCode($next) : $previous_block; + $blocks[$block] = ($blocks[$block] ?? '') . $character; } else { - $non_latin .= $character; + $block = self::getBlockCode($character); + $blocks[$block] = ($blocks[$block] ?? '') . $character; + } + } + + foreach (array_keys($blocks) as $key) { + $blocks[$key] = trim($blocks[$key]); + if (empty($blocks[$key])) { + unset($blocks[$key]); } } - return (mb_strlen($latin) > mb_strlen($non_latin)) ? $latin : $non_latin; + + return array_values($blocks); + } + + /** + * returns the block code for the given character + * + * @param string $character + * @return integer 0 = no alpha character (blank, signs, emojis, ...), 1 = latin character, 2 = character in every other language + */ + private static function getBlockCode(string $character): int + { + if (!\IntlChar::isalpha($character)) { + return 0; + } + return self::isLatin($character) ? 1 : 2; + } + + /** + * Checks if the given character is in one of the latin code blocks + * + * @param string $character + * @return boolean + */ + private static function isLatin(string $character): bool + { + return in_array(\IntlChar::getBlockCode($character), [ + \IntlChar::BLOCK_CODE_BASIC_LATIN, \IntlChar::BLOCK_CODE_LATIN_1_SUPPLEMENT, + \IntlChar::BLOCK_CODE_LATIN_EXTENDED_A, \IntlChar::BLOCK_CODE_LATIN_EXTENDED_B, + \IntlChar::BLOCK_CODE_LATIN_EXTENDED_C, \IntlChar::BLOCK_CODE_LATIN_EXTENDED_D, + \IntlChar::BLOCK_CODE_LATIN_EXTENDED_E, \IntlChar::BLOCK_CODE_LATIN_EXTENDED_ADDITIONAL + ]); } public static function getLanguageMessage(array $item): string @@ -2079,7 +2166,15 @@ class Item $used_languages = ''; foreach (json_decode($item['language'], true) as $language => $reliability) { - $used_languages .= $iso639->languageByCode1($language) . ' (' . $language . "): " . number_format($reliability, 5) . '\n'; + $code = DI::l10n()->toISO6391($language); + + $native = $iso639->nativeByCode1($code); + $language = $iso639->languageByCode1($code); + if ($native != $language) { + $used_languages .= DI::l10n()->t('%s (%s - %s): %s', $native, $language, $code, number_format($reliability, 5)) . '\n'; + } else { + $used_languages .= DI::l10n()->t('%s (%s): %s', $native, $code, number_format($reliability, 5)) . '\n'; + } } $used_languages = DI::l10n()->t('Detected languages in this post:\n%s', $used_languages); return $used_languages; @@ -2107,7 +2202,7 @@ class Item $hostPart = $host ?: $parsed['host'] ?? ''; if (!$hostPart) { - Logger::warning('Empty host GUID part', ['uri' => $uri, 'host' => $host, 'parsed' => $parsed, 'callstack' => System::callstack(10)]); + Logger::warning('Empty host GUID part', ['uri' => $uri, 'host' => $host, 'parsed' => $parsed]); } // Glue it together to be able to make a hash from it @@ -3428,11 +3523,8 @@ class Item continue; } - if ($PostMedia->mimetype->type == 'image') { - $preview_size = $PostMedia->width > $PostMedia->height ? Proxy::SIZE_MEDIUM : Proxy::SIZE_LARGE; - $preview_url = DI::baseUrl() . $PostMedia->getPreviewPath($preview_size); - } elseif ($PostMedia->preview) { - $preview_size = Proxy::SIZE_LARGE; + if ($PostMedia->mimetype->type == 'image' || $PostMedia->preview) { + $preview_size = Proxy::SIZE_MEDIUM; $preview_url = DI::baseUrl() . $PostMedia->getPreviewPath($preview_size); } else { $preview_size = 0; @@ -3853,11 +3945,12 @@ class Item * Fetches item for given URI or plink * * @param string $uri - * @param integer $uid + * @param int $uid + * @param int $completion * * @return integer item id */ - public static function fetchByLink(string $uri, int $uid = 0): int + public static function fetchByLink(string $uri, int $uid = 0, int $completion = ActivityPub\Receiver::COMPLETION_MANUAL): int { Logger::info('Trying to fetch link', ['uid' => $uid, 'uri' => $uri]); $item_id = self::searchByLink($uri, $uid); @@ -3878,7 +3971,7 @@ class Item return is_numeric($hookData['item_id']) ? $hookData['item_id'] : 0; } - $fetched_uri = ActivityPub\Processor::fetchMissingActivity($uri, [], '', ActivityPub\Receiver::COMPLETION_MANUAL, $uid); + $fetched_uri = ActivityPub\Processor::fetchMissingActivity($uri, [], '', $completion, $uid); if ($fetched_uri) { $item_id = self::searchByLink($fetched_uri, $uid); @@ -3938,7 +4031,7 @@ class Item } $url = $shared['message_id'] ?: $shared['link']; - $id = self::fetchByLink($url); + $id = self::fetchByLink($url, 0, ActivityPub\Receiver::COMPLETION_ASYNC); if (!$id) { Logger::notice('Post could not be fetched.', ['url' => $url, 'uid' => $uid]); return 0;