-- ------------------------------------------
-- Friendica 2022.09-dev (Giant Rhubarb)
--- DB_UPDATE_VERSION 1474
+-- DB_UPDATE_VERSION 1475
-- ------------------------------------------
`featured-tags` varchar(255) COMMENT 'Address for the collection of featured tags',
`manually-approve` boolean COMMENT '',
`discoverable` boolean COMMENT 'Mastodon extension: true if profile is published in their directory',
+ `suspended` boolean COMMENT 'Mastodon extension: true if profile is suspended',
`nick` varchar(255) NOT NULL DEFAULT '' COMMENT '',
`name` varchar(255) COMMENT '',
`about` text COMMENT '',
FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='private messages';
---
--- TABLE conversation
---
-CREATE TABLE IF NOT EXISTS `conversation` (
- `item-uri` varbinary(255) NOT NULL COMMENT 'Original URI of the item - unrelated to the table with the same name',
- `reply-to-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'URI to which this item is a reply',
- `conversation-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation URI',
- `conversation-href` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation link',
- `protocol` tinyint unsigned NOT NULL DEFAULT 255 COMMENT 'The protocol of the item',
- `direction` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'How the message arrived here: 1=push, 2=pull',
- `source` mediumtext COMMENT 'Original source',
- `received` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Receiving date',
- PRIMARY KEY(`item-uri`),
- INDEX `conversation-uri` (`conversation-uri`),
- INDEX `received` (`received`)
-) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Raw data and structure information for messages';
-
--
-- TABLE workerqueue
--
FOREIGN KEY (`vid`) REFERENCES `verb` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Structure for all posts';
+--
+-- TABLE post-activity
+--
+CREATE TABLE IF NOT EXISTS `post-activity` (
+ `uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
+ `activity` mediumtext COMMENT 'Original activity',
+ `received` datetime COMMENT '',
+ PRIMARY KEY(`uri-id`),
+ FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Original remote activity';
+
--
-- TABLE post-category
--
| [contact](help/database/db_contact) | contact table |
| [contact-relation](help/database/db_contact-relation) | Contact relations |
| [conv](help/database/db_conv) | private messages |
-| [conversation](help/database/db_conversation) | Raw data and structure information for messages |
| [delayed-post](help/database/db_delayed-post) | Posts that are about to be distributed at a later time |
| [diaspora-interaction](help/database/db_diaspora-interaction) | Signed Diaspora Interaction |
| [endpoint](help/database/db_endpoint) | ActivityPub endpoints - used in the ActivityPub implementation |
| [permissionset](help/database/db_permissionset) | |
| [photo](help/database/db_photo) | photo storage |
| [post](help/database/db_post) | Structure for all posts |
+| [post-activity](help/database/db_post-activity) | Original remote activity |
| [post-category](help/database/db_post-category) | post relation to categories |
| [post-collection](help/database/db_post-collection) | Collection of posts |
| [post-content](help/database/db_post-content) | Content for all posts |
| featured-tags | Address for the collection of featured tags | varchar(255) | YES | | NULL | |
| manually-approve | | boolean | YES | | NULL | |
| discoverable | Mastodon extension: true if profile is published in their directory | boolean | YES | | NULL | |
+| suspended | Mastodon extension: true if profile is suspended | boolean | YES | | NULL | |
| nick | | varchar(255) | NO | | | |
| name | | varchar(255) | YES | | NULL | |
| about | | text | YES | | NULL | |
--- /dev/null
+Table post-activity
+===========
+
+Original remote activity
+
+Fields
+------
+
+| Field | Description | Type | Null | Key | Default | Extra |
+| -------- | --------------------------------------------------------- | ------------ | ---- | --- | ------- | ----- |
+| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | |
+| activity | Original activity | mediumtext | YES | | NULL | |
+| received | | datetime | YES | | NULL | |
+
+Indexes
+------------
+
+| Name | Fields |
+| ------- | ------ |
+| PRIMARY | uri-id |
+
+Foreign Keys
+------------
+
+| Field | Target Table | Target Field |
+|-------|--------------|--------------|
+| uri-id | [item-uri](help/database/db_item-uri) | id |
+
+Return to [database documentation](help/database)
$datarray['protocol'] = Conversation::PARCEL_DIRECT;
$datarray['direction'] = Conversation::PUSH;
- $conversation = DBA::selectFirst('conversation', ['conversation-uri', 'conversation-href'], ['item-uri' => $datarray['thr-parent']]);
- if (DBA::isResult($conversation)) {
- if ($conversation['conversation-uri'] != '') {
- $datarray['conversation-uri'] = $conversation['conversation-uri'];
- }
- if ($conversation['conversation-href'] != '') {
- $datarray['conversation-href'] = $conversation['conversation-href'];
- }
- }
-
if ($orig_post) {
$datarray['edit'] = true;
} else {
$old_tables = ['fserver', 'gcign', 'gcontact', 'gcontact-relation', 'gfollower' ,'glink', 'item-delivery-data',
'item-activity', 'item-content', 'item_id', 'participation', 'poll', 'poll_result', 'queue', 'retriever_rule',
'deliverq', 'dsprphotoq', 'ffinder', 'sign', 'spam', 'term', 'user-item', 'thread', 'item', 'challenge',
- 'auth_codes', 'tokens', 'clients', 'profile_check', 'host'];
+ 'auth_codes', 'tokens', 'clients', 'profile_check', 'host', 'conversation'];
$tables = DBA::selectToArray('INFORMATION_SCHEMA.TABLES', ['TABLE_NAME'],
['TABLE_SCHEMA' => DBA::databaseName(), 'TABLE_TYPE' => 'BASE TABLE']);
use Friendica\Util\HTTPSignature;
use Friendica\Util\JsonLD;
use Friendica\Util\Network;
+use GuzzleHttp\Psr7\Uri;
class APContact
{
$apcontact['manually-approve'] = (int)JsonLD::fetchElement($compacted, 'as:manuallyApprovesFollowers');
+ $apcontact['suspended'] = (int)JsonLD::fetchElement($compacted, 'toot:suspended');
+
if (!empty($compacted['as:generator'])) {
$apcontact['baseurl'] = JsonLD::fetchElement($compacted['as:generator'], 'as:url', '@id');
$apcontact['generator'] = JsonLD::fetchElement($compacted['as:generator'], 'as:name', '@value');
if (strlen($apcontact['photo']) > 255) {
$parts = parse_url($apcontact['photo']);
unset($parts['fragment']);
- $apcontact['photo'] = Network::unparseURL($parts);
+ $apcontact['photo'] = Uri::fromParts($parts);
if (strlen($apcontact['photo']) > 255) {
unset($parts['query']);
- $apcontact['photo'] = Network::unparseURL($parts);
+ $apcontact['photo'] = Uri::fromParts($parts);
}
if (strlen($apcontact['photo']) > 255) {
namespace Friendica\Model;
-use Friendica\Core\Protocol;
-use Friendica\Database\Database;
-use Friendica\Database\DBA;
-use Friendica\Util\DateTimeFormat;
-
class Conversation
{
/*
*/
const RELAY = 3;
- public static function getByItemUri(string $item_uri)
- {
- return DBA::selectFirst('conversation', [], ['item-uri' => $item_uri]);
- }
-
- /**
- * Store the conversation data
- *
- * @param array $arr Item array with conversation data
- * @return array Item array with removed conversation data
- * @throws \Exception
- */
- public static function insert(array $arr): array
- {
- if (in_array(($arr['network'] ?? '') ?: Protocol::PHANTOM,
- [Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, Protocol::TWITTER]) && !empty($arr['uri'])) {
- $conversation = ['item-uri' => $arr['uri'], 'received' => DateTimeFormat::utcNow()];
-
- if (isset($arr['parent-uri']) && ($arr['parent-uri'] != $arr['uri'])) {
- $conversation['reply-to-uri'] = $arr['parent-uri'];
- }
-
- if (isset($arr['thr-parent']) && ($arr['thr-parent'] != $arr['uri'])) {
- $conversation['reply-to-uri'] = $arr['thr-parent'];
- }
-
- if (isset($arr['conversation-uri'])) {
- $conversation['conversation-uri'] = $arr['conversation-uri'];
- }
-
- if (isset($arr['conversation-href'])) {
- $conversation['conversation-href'] = $arr['conversation-href'];
- }
-
- if (isset($arr['protocol'])) {
- $conversation['protocol'] = $arr['protocol'];
- }
-
- if (isset($arr['direction'])) {
- $conversation['direction'] = $arr['direction'];
- }
-
- if (isset($arr['source'])) {
- $conversation['source'] = $arr['source'];
- }
-
- if (!DBA::exists('conversation', ['item-uri' => $conversation['item-uri']])) {
- DBA::insert('conversation', $conversation, Database::INSERT_IGNORE);
- }
- }
-
- unset($arr['conversation-uri']);
- unset($arr['conversation-href']);
- unset($arr['source']);
-
- return $arr;
- }
}
$serverdata['registered-users'] = $serverdata['registered-users'] ?? 0;
+ // Numbers above a reasonable value (10 millions) are ignored
+ if ($serverdata['registered-users'] > 10000000) {
+ $serverdata['registered-users'] = 0;
+ }
+
// On an active server there has to be at least a single user
- if (!in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED]) && ($serverdata['registered-users'] == 0)) {
+ if (!in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED]) && ($serverdata['registered-users'] <= 0)) {
$serverdata['registered-users'] = 1;
} elseif (in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED])) {
$serverdata['registered-users'] = 0;
$item['parent-uri-id'] = ItemURI::getIdByURI($item['parent-uri']);
// Store conversation data
- $item = Conversation::insert($item);
+ $source = $item['source'] ?? '';
+ unset($item['conversation-uri']);
+ unset($item['conversation-href']);
+ unset($item['source']);
/*
* Do we already have this item?
}
if ($transmit) {
+ if (!empty($source)) {
+ Post\Activity::insert($item['uri-id'], $source);
+ }
Worker::add(['priority' => $priority, 'dont_fork' => true], 'Notifier', $notify_type, (int)$posted_item['uri-id'], (int)$posted_item['uid']);
}
--- /dev/null
+<?php
+/**
+ * @copyright Copyright (C) 2010-2022, the Friendica project
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Model\Post;
+
+use Friendica\Database\Database;
+use Friendica\Database\DBA;
+use Friendica\Util\DateTimeFormat;
+
+class Activity
+{
+ /**
+ * Insert a new post-activity entry
+ *
+ * @param integer $uri_id
+ * @param array $fields
+ *
+ * @return bool success
+ */
+ public static function insert(int $uri_id, string $source): bool
+ {
+ // Additionally assign the key fields
+ $fields = [
+ 'uri-id' => $uri_id,
+ 'activity' => $source,
+ 'received' => DateTimeFormat::utcNow()
+ ];
+
+ return DBA::insert('post-activity', $fields, Database::INSERT_IGNORE);
+ }
+
+ /**
+ * Retrieves activity of the given uri-id
+ *
+ * @param int $uriId
+ *
+ * @return array
+ */
+ public static function getByURIId(int $uriId): array
+ {
+ $activity = DBA::selectFirst('post-activity', [], ['uri-id' => $uriId]);
+ return json_decode($activity['activity'] ?? '', true) ?? [];
+ }
+
+ /**
+ * Checks if the given uridid has a stored activity
+ *
+ * @param integer $uriId
+ *
+ * @return boolean
+ */
+ public static function exists(int $uriId): bool
+ {
+ return DBA::exists('post-activity', ['uri-id' => $uriId]);
+ }
+}
break;
default:
- Logger:warning('Unknown tag type found', $tag);
+ Logger::warning('Unknown tag type found', $tag);
}
}
DBA::close($taglist);
$guid = basename($_REQUEST['guid'] ?? $this->parameters['guid'] ?? '');
- $source = '';
$item_uri = '';
$item_id = '';
$terms = [];
$item = Model\Post::selectFirst(['id', 'uri-id', 'guid', 'uri'], ['guid' => $guid]);
if ($item) {
- $conversation = Model\Conversation::getByItemUri($item['uri']);
-
$item_id = $item['id'];
$item_uri = $item['uri'];
- $source = $conversation['source'];
$terms = Model\Tag::getByURIId($item['uri-id'], [Model\Tag::HASHTAG, Model\Tag::MENTION, Model\Tag::IMPLICIT_MENTION]);
}
}
$o = Renderer::replaceMacros($tpl, [
'$title' => DI::l10n()->t('Item Source'),
'$guid' => ['guid', DI::l10n()->t('Item Guid'), $guid, ''],
- '$source' => $source,
'$item_uri' => $item_uri,
'$item_id' => $item_id,
'$terms' => $terms,
'$urllbl' => DI::l10n()->t('URL'),
'$mentionlbl' => DI::l10n()->t('Mention'),
'$implicitlbl' => DI::l10n()->t('Implicit Mention'),
- '$sourcelbl' => DI::l10n()->t('Source'),
]);
return $o;
}
}
- private static function dispatchPublic(array $postdata)
+ private static function dispatchPublic(string $postdata)
{
$msg = Diaspora::decodeRaw($postdata, '', true);
if (!is_array($msg)) {
}
if (($Notification->verb == Activity::POST) || ($Notification->type === Post\UserNotification::TYPE_SHARED)) {
- $item = Post::selectFirst([], ['uri-id' => $item['thr-parent-id'], 'uid' => [0, $Notification->uid]], ['order' => ['uid' => true]]);
+ $thrparentid = $item['thr-parent-id'];
+ $item = Post::selectFirst([], ['uri-id' => $thrparentid, 'uid' => [0, $Notification->uid]], ['order' => ['uid' => true]]);
if (empty($item)) {
- $this->logger->info('Thread parent post not found', ['uri-id' => $item['thr-parent-id']]);
+ $this->logger->info('Thread parent post not found', ['uri-id' => $thrparentid]);
return $message;
}
}
} elseif ($cmd == WorkerDelivery::PROFILEUPDATE) {
$success = ActivityPub\Transmitter::sendProfileUpdate($uid, $inbox);
} else {
- $data = ActivityPub\Transmitter::createCachedActivityFromItem($item_id);
+ $data = Post\Activity::getByURIId($uri_id);
+ if (empty($data)) {
+ $data = ActivityPub\Transmitter::createCachedActivityFromItem($item_id);
+ }
if (!empty($data)) {
$timestamp = microtime(true);
$response = HTTPSignature::post($data, $inbox, $uid);
if (!empty($activity['raw'])) {
$item['source'] = $activity['raw'];
$item['protocol'] = Conversation::PARCEL_ACTIVITYPUB;
- $item['conversation-href'] = $activity['context'] ?? '';
- $item['conversation-uri'] = $activity['conversation'] ?? '';
if (isset($activity['push'])) {
$item['direction'] = $activity['push'] ? Conversation::PUSH : Conversation::PULL;
}
// @todo To ensure that the remote system is working correctly, we can check if the "Content-Type" contains JSON
- return in_array($curlResult->getReturnCode(), [404]);
+ if (in_array($curlResult->getReturnCode(), [404])) {
+ return true;
+ }
+
+ $object = json_decode($curlResult->getBody(), true);
+ if (!empty($object)) {
+ $activity = JsonLD::compact($object);
+ if (JsonLD::fetchElement($activity, '@type') == 'as:Tombstone') {
+ return true;
+ }
+ }
+
+ return false;
}
/**
* Delete items
use Friendica\Database\Database;
use Friendica\Database\DBA;
use Friendica\DI;
+use Friendica\Protocol\ActivityPub;
use Friendica\Util\DateTimeFormat;
+use Friendica\Util\JsonLD;
/**
* This class handles the processing of incoming posts
Logger::debug('Process leftover entry', $entry);
self::process($entry['id']);
}
+ DBA::close($entries);
}
/**
while ($entry = DBA::fetch($entries)) {
self::process($entry['id']);
}
+ DBA::close($entries);
+ }
+
+ /**
+ * Prepare the queue entry.
+ * This is a test function that is used solely for development.
+ *
+ * @param integer $id
+ * @return array
+ */
+ public static function reprepareActivityById(int $id): array
+ {
+ $entry = DBA::selectFirst('inbox-entry', [], ['id' => $id]);
+ if (empty($entry)) {
+ return [];
+ }
+
+ $receiver = DBA::selectFirst('inbox-entry-receiver', ['uid'], ['queue-id' => $id]);
+ if (!empty($receiver)) {
+ $uid = $receiver['uid'];
+ } else {
+ $uid = 0;
+ }
+
+ $trust_source = $entry['trust'];
+
+ $data = json_decode($entry['activity'], true);
+ $activity = json_decode($data['raw'], true);
+
+ $ldactivity = JsonLD::compact($activity);
+ return [
+ 'data' => Receiver::prepareObjectData($ldactivity, $uid, $entry['push'], $trust_source),
+ 'trust' => $trust_source
+ ];
+ }
+
+ /**
+ * Set the trust for all untrusted entries.
+ * This is a test function that is used solely for development.
+ *
+ * @return void
+ */
+ public static function reprepareAll()
+ {
+ $entries = DBA::select('inbox-entry', ['id'], ["NOT `trust` AND `wid` IS NULL"], ['order' => ['id' => true]]);
+ while ($entry = DBA::fetch($entries)) {
+ $data = self::reprepareActivityById($entry['id'], false);
+ if ($data['trust']) {
+ DBA::update('inbox-entry', ['trust' => true], ['id' => $entry['id']]);
+ }
+ }
+ DBA::close($entries);
+
}
}
*/
public static function prepareObjectData(array $activity, int $uid, bool $push, bool &$trust_source): array
{
- $id = JsonLD::fetchElement($activity, '@id');
+ $id = JsonLD::fetchElement($activity, '@id');
+ $type = JsonLD::fetchElement($activity, '@type');
$object_id = JsonLD::fetchElement($activity, 'as:object', '@id');
+ if (!empty($object_id) && in_array($type, ['as:Create', 'as:Update'])) {
+ $fetch_id = $object_id;
+ } else {
+ $fetch_id = $id;
+ }
+
+ if (!empty($activity['as:object'])) {
+ $object_type = JsonLD::fetchElement($activity['as:object'], '@type');
+ }
+
if (!empty($id) && !$trust_source) {
$fetch_uid = $uid ?: self::getBestUserForActivity($activity);
- $fetched_activity = ActivityPub::fetchContent($id, $fetch_uid);
+ $fetched_activity = ActivityPub::fetchContent($fetch_id, $fetch_uid);
if (!empty($fetched_activity)) {
$object = JsonLD::compact($fetched_activity);
- $fetched_id = JsonLD::fetchElement($object, '@id');
- if ($fetched_id == $id) {
+
+ $fetched_id = JsonLD::fetchElement($object, '@id');
+ $fetched_type = JsonLD::fetchElement($object, '@type');
+
+ if (($fetched_id == $id) && !empty($fetched_type) && ($fetched_type == $type)) {
Logger::info('Activity had been fetched successfully', ['id' => $id]);
$trust_source = true;
- if ($id != $object_id) {
- $activity = $object;
- } else {
- Logger::info('Fetched data is the object instead of the activity', ['id' => $id]);
- unset($object['@context']);
- $activity['as:object'] = $object;
- }
+ $activity = $object;
+ } elseif (($fetched_id == $object_id) && !empty($fetched_type) && ($fetched_type == $object_type)) {
+ Logger::info('Fetched data is the object instead of the activity', ['id' => $id]);
+ $trust_source = true;
+ unset($object['@context']);
+ $activity['as:object'] = $object;
} else {
Logger::info('Activity id is not equal', ['id' => $id, 'fetched' => $fetched_id]);
}
$object_data['object_object'] = JsonLD::fetchElement($activity['as:object'], 'as:object');
$object_data['object_type'] = JsonLD::fetchElement($activity['as:object'], '@type');
$object_data['push'] = $push;
- if ($type == 'as:Delete') {
+ if (!$trust_source && ($type == 'as:Delete')) {
$apcontact = APContact::getByURL($object_data['object_id'], true);
- $trust_source = ($apcontact['type'] == 'Tombstone');
+ $trust_source = empty($apcontact) || ($apcontact['type'] == 'Tombstone') || $apcontact['suspended'];
}
} elseif (in_array($type, ['as:Create', 'as:Update', 'as:Announce', 'as:Invite']) || strpos($type, '#emojiReaction')) {
// Fetch the content only on activities where this matters
$object_data['object_object_type'] = self::fetchObjectType([], $object_data['object_object'], $fetch_uid);
}
- if (($type == 'as:Delete') && in_array($object_data['object_type'], array_merge(['as:Tombstone'], self::CONTENT_TYPES))) {
+ if (!$trust_source && ($type == 'as:Delete') && in_array($object_data['object_type'], array_merge(['as:Tombstone', ''], self::CONTENT_TYPES))) {
$trust_source = Processor::isActivityGone($object_data['object_id']);
+ if (!$trust_source) {
+ $trust_source = !empty(APContact::getByURL($object_data['object_id'], false));
+ }
}
}
if (!empty($object_data['raw'])) {
$announce_object_data['raw'] = $object_data['raw'];
}
+ if (!empty($object_data['raw-object'])) {
+ $announce_object_data['raw-object'] = $object_data['raw-object'];
+ }
ActivityPub\Processor::createActivity($announce_object_data, Activity::ANNOUNCE);
}
} else {
$object_data = self::processObject($object);
if (!empty($data)) {
- $object_data['raw'] = json_encode($data);
+ $object_data['raw-object'] = json_encode($data);
}
return $object_data;
}
}
if (!$item['deleted']) {
- $condition = ['item-uri' => $item['uri'], 'protocol' => Conversation::PARCEL_ACTIVITYPUB];
- $conversation = DBA::selectFirst('conversation', ['source'], $condition);
- if (!$item['origin'] && DBA::isResult($conversation)) {
- $data = json_decode($conversation['source'], true);
+ $data = Post\Activity::getByURIId($item['uri-id']);
+ if (!$item['origin'] && !empty($data)) {
if (!empty($data['type'])) {
if (in_array($data['type'], ['Create', 'Update'])) {
if ($object_mode) {
}
// Add conversation data. This is used for OStatus
- $conversation_href = DI::baseUrl()."/display/".$item["parent-guid"];
- $conversation_uri = $conversation_href;
-
- if (isset($parent_item)) {
- $conversation = DBA::selectFirst('conversation', ['conversation-uri', 'conversation-href'], ['item-uri' => $item['thr-parent']]);
- if (DBA::isResult($conversation)) {
- if ($conversation['conversation-uri'] != '') {
- $conversation_uri = $conversation['conversation-uri'];
- }
- if ($conversation['conversation-href'] != '') {
- $conversation_href = $conversation['conversation-href'];
- }
- }
- }
-
$attributes = [
- 'href' => $conversation_href,
- 'ref' => $conversation_uri,
+ 'href' => $item['conversation'],
+ 'ref' => $item['conversation'],
];
- XML::addElement($doc, $entry, 'ostatus:conversation', $conversation_uri, $attributes);
+ XML::addElement($doc, $entry, 'ostatus:conversation', $item['conversation'], $attributes);
XML::addElement($doc, $entry, 'id', $item['uri']);
XML::addElement($doc, $entry, 'title', $item['title']);
self::parseLinks($links, $item);
}
- $item['conversation-uri'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry);
+ $item['conversation'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry);
$conv = $xpath->query('ostatus:conversation', $entry);
if (is_object($conv->item(0))) {
foreach ($conv->item(0)->attributes as $attributes) {
if ($attributes->name == 'ref') {
- $item['conversation-uri'] = $attributes->textContent;
+ $item['conversation'] = $attributes->textContent;
}
if ($attributes->name == 'href') {
- $item['conversation-href'] = $attributes->textContent;
+ $item['conversation'] = $attributes->textContent;
}
}
}
$item['created'] = XML::getFirstNodeValue($xpath, 'atom:published/text()', $entry);
$item['edited'] = XML::getFirstNodeValue($xpath, 'atom:updated/text()', $entry);
- $item['conversation-uri'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry);
+ $item['conversation'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry);
$conv = $xpath->query('ostatus:conversation', $entry);
if (is_object($conv->item(0))) {
foreach ($conv->item(0)->attributes as $attributes) {
if ($attributes->name == 'ref') {
- $item['conversation-uri'] = $attributes->textContent;
+ $item['conversation'] = $attributes->textContent;
}
if ($attributes->name == 'href') {
- $item['conversation-href'] = $attributes->textContent;
+ $item['conversation'] = $attributes->textContent;
}
}
}
}
}
- if (($self != '') && empty($item['protocol'])) {
- self::fetchSelf($self, $item);
- }
-
- if (!empty($item['conversation-href'])) {
- self::fetchConversation($item['conversation-href'], $item['conversation-uri']);
- }
-
if (isset($item['thr-parent'])) {
if (!Post::exists(['uid' => $importer['uid'], 'uri' => $item['thr-parent']])) {
if ($related != '') {
$item['gravity'] = GRAVITY_PARENT;
}
- if (($item['author-link'] != '') && !empty($item['protocol'])) {
- $item = Conversation::insert($item);
- }
-
self::$itemlist[] = $item;
}
- /**
- * Fetch the conversation for posts
- *
- * @param string $conversation The link to the conversation
- * @param string $conversation_uri The conversation in "uri" format
- * @return void
- * @throws \Friendica\Network\HTTPException\InternalServerErrorException
- */
- private static function fetchConversation(string $conversation, string $conversation_uri)
- {
- // Ensure that we only store a conversation once in a process
- if (isset(self::$conv_list[$conversation])) {
- return;
- }
-
- self::$conv_list[$conversation] = true;
-
- $curlResult = DI::httpClient()->get($conversation, HttpClientAccept::ATOM_XML);
-
- if (!$curlResult->isSuccess() || empty($curlResult->getBody())) {
- return;
- }
-
- $xml = '';
-
- if ($curlResult->inHeader('Content-Type') &&
- in_array('application/atom+xml', $curlResult->getHeader('Content-Type'))) {
- $xml = $curlResult->getBody();
- }
-
- if ($xml == '') {
- $doc = new DOMDocument();
- if (!@$doc->loadHTML($curlResult->getBody())) {
- return;
- }
- $xpath = new DOMXPath($doc);
-
- $links = $xpath->query('//link');
- if ($links) {
- $file = '';
- foreach ($links as $link) {
- $attribute = self::readAttributes($link);
- if (($attribute['rel'] == 'alternate') && ($attribute['type'] == 'application/atom+xml')) {
- $file = $attribute['href'];
- }
- }
- if ($file != '') {
- $conversation_atom = DI::httpClient()->get($attribute['href'], HttpClientAccept::ATOM_XML);
-
- if ($conversation_atom->isSuccess()) {
- $xml = $conversation_atom->getBody();
- }
- }
- }
- }
-
- if ($xml == '') {
- return;
- }
-
- self::storeConversation($xml, $conversation, $conversation_uri);
- }
-
- /**
- * Store a feed in several conversation entries
- *
- * @param string $xml The feed
- * @param string $conversation conversation
- * @param string $conversation_uri conversation uri
- * @return void
- * @throws \Exception
- */
- private static function storeConversation(string $xml, string $conversation = '', string $conversation_uri = '')
- {
- $doc = new DOMDocument();
- @$doc->loadXML($xml);
-
- $xpath = new DOMXPath($doc);
- $xpath->registerNamespace('atom', ActivityNamespace::ATOM1);
- $xpath->registerNamespace('thr', ActivityNamespace::THREAD);
- $xpath->registerNamespace('ostatus', ActivityNamespace::OSTATUS);
-
- $entries = $xpath->query('/atom:feed/atom:entry');
-
- // Now store the entries
- foreach ($entries as $entry) {
- $doc2 = new DOMDocument();
- $doc2->preserveWhiteSpace = false;
- $doc2->formatOutput = true;
-
- $conv_data = [];
-
- $conv_data['protocol'] = Conversation::PARCEL_SPLIT_CONVERSATION;
- $conv_data['direction'] = Conversation::PULL;
- $conv_data['network'] = Protocol::OSTATUS;
- $conv_data['uri'] = XML::getFirstNodeValue($xpath, 'atom:id/text()', $entry);
-
- $inreplyto = $xpath->query('thr:in-reply-to', $entry);
- if (is_object($inreplyto->item(0))) {
- foreach ($inreplyto->item(0)->attributes as $attributes) {
- if ($attributes->name == 'ref') {
- $conv_data['reply-to-uri'] = $attributes->textContent;
- }
- }
- }
-
- $conv_data['conversation-uri'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry);
-
- $conv = $xpath->query('ostatus:conversation', $entry);
- if (is_object($conv->item(0))) {
- foreach ($conv->item(0)->attributes as $attributes) {
- if ($attributes->name == 'ref') {
- $conv_data['conversation-uri'] = $attributes->textContent;
- }
- if ($attributes->name == 'href') {
- $conv_data['conversation-href'] = $attributes->textContent;
- }
- }
- }
-
- if ($conversation != '') {
- $conv_data['conversation-uri'] = $conversation;
- }
-
- if ($conversation_uri != '') {
- $conv_data['conversation-uri'] = $conversation_uri;
- }
-
- $entry = $doc2->importNode($entry, true);
-
- $doc2->appendChild($entry);
-
- $conv_data['source'] = $doc2->saveXML();
-
- Logger::info('Store conversation data for uri '.$conv_data['uri']);
- Conversation::insert($conv_data);
- }
- }
-
- /**
- * Fetch the own post so that it can be stored later
- *
- * We want to store the original data for later processing.
- * This function is meant for cases where we process a feed with multiple entries.
- * In that case we need to fetch the single posts here.
- *
- * @param string $self The link to the self item
- * @param array $item The item array
- * @return void
- * @throws \Friendica\Network\HTTPException\InternalServerErrorException
- */
- private static function fetchSelf(string $self, array &$item)
- {
- $condition = ['item-uri' => $self, 'protocol' => [Conversation::PARCEL_DFRN,
- Conversation::PARCEL_DIASPORA_DFRN, Conversation::PARCEL_LOCAL_DFRN,
- Conversation::PARCEL_DIRECT, Conversation::PARCEL_SALMON]];
- if (DBA::exists('conversation', $condition)) {
- Logger::info('Conversation '.$item['uri'].' is already stored.');
- return;
- }
-
- $curlResult = DI::httpClient()->get($self, HttpClientAccept::ATOM_XML);
-
- if (!$curlResult->isSuccess()) {
- return;
- }
-
- // We reformat the XML to make it better readable
- $doc = new DOMDocument();
- $doc->loadXML($curlResult->getBody());
- $doc->preserveWhiteSpace = false;
- $doc->formatOutput = true;
- $xml = $doc->saveXML();
-
- $item['protocol'] = Conversation::PARCEL_SALMON;
- $item['source'] = $xml;
- $item['direction'] = Conversation::PULL;
-
- Logger::info('Conversation '.$item['uri'].' is now fetched.');
- }
-
/**
* Fetch related posts and processes them
*
*/
private static function fetchRelated(string $related, string $related_uri, array $importer)
{
- $condition = [
- 'item-uri' => $related_uri,
- 'protocol' => [
- Conversation::PARCEL_DFRN,
- Conversation::PARCEL_DIASPORA_DFRN,
- Conversation::PARCEL_LOCAL_DFRN,
- Conversation::PARCEL_DIRECT,
- Conversation::PARCEL_SALMON,
- ],
- ];
- $conversation = DBA::selectFirst('conversation', ['source', 'protocol'], $condition);
- if (DBA::isResult($conversation)) {
- $stored = true;
- $xml = $conversation['source'];
- if (self::process($xml, $importer, $contact, $hub, $stored, false, Conversation::PULL)) {
- Logger::info('Got valid cached XML for URI '.$related_uri);
- return;
- }
- if ($conversation['protocol'] == Conversation::PARCEL_SALMON) {
- Logger::info('Delete invalid cached XML for URI '.$related_uri);
- DBA::delete('conversation', ['item-uri' => $related_uri]);
- }
- }
-
$stored = false;
$curlResult = DI::httpClient()->get($related, HttpClientAccept::ATOM_XML);
}
}
- // Finally we take the data that we fetched from "ostatus:conversation"
- if ($xml == '') {
- $condition = ['item-uri' => $related_uri, 'protocol' => Conversation::PARCEL_SPLIT_CONVERSATION];
- $conversation = DBA::selectFirst('conversation', ['source'], $condition);
- if (DBA::isResult($conversation)) {
- $stored = true;
- Logger::info('Got cached XML from conversation for URI ' . $related_uri);
- $xml = $conversation['source'];
- }
- }
-
if ($xml != '') {
self::process($xml, $importer, $contact, $hub, $stored, false, Conversation::PULL);
} else {
case 'ostatus:conversation':
$link_data['conversation'] = $attribute['href'];
- $item['conversation-href'] = $link_data['conversation'];
- if (!isset($item['conversation-uri'])) {
- $item['conversation-uri'] = $item['conversation-href'];
- }
+ $item['conversation'] = $link_data['conversation'];
break;
case 'enclosure':
}
if (intval($item['parent']) > 0) {
- $conversation_href = $conversation_uri = str_replace('/objects/', '/context/', $item['thr-parent']);
-
- if (isset($parent_item)) {
- $conversation = DBA::selectFirst('conversation', ['conversation-uri', 'conversation-href'], ['item-uri' => $parent_item]);
- if (DBA::isResult($conversation)) {
- if ($conversation['conversation-uri'] != '') {
- $conversation_uri = $conversation['conversation-uri'];
- }
- if ($conversation['conversation-href'] != '') {
- $conversation_href = $conversation['conversation-href'];
- }
- }
- }
+ $conversation_href = $conversation_uri = $item['conversation'];
XML::addElement($doc, $entry, 'link', '', ['rel' => 'ostatus:conversation', 'href' => $conversation_href]);
Worker::add(PRIORITY_LOW, 'ExpirePosts');
- Worker::add(PRIORITY_LOW, 'ExpireConversations');
+ Worker::add(PRIORITY_LOW, 'ExpireActivities');
Worker::add(PRIORITY_LOW, 'RemoveUnusedTags');
--- /dev/null
+<?php
+/**
+ * @copyright Copyright (C) 2010-2022, the Friendica project
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Worker;
+
+use Friendica\Database\DBA;
+use Friendica\Util\DateTimeFormat;
+
+class ExpireActivities
+{
+ /**
+ * Delete old post-activity entries
+ */
+ public static function execute()
+ {
+ DBA::delete('post-activity', ["`received` < ?", DateTimeFormat::utc('now - 7 days')]);
+ }
+}
+++ /dev/null
-<?php
-/**
- * @copyright Copyright (C) 2010-2022, the Friendica project
- *
- * @license GNU AGPL version 3 or any later version
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
- */
-
-namespace Friendica\Worker;
-
-use Friendica\Database\DBA;
-use Friendica\DI;
-use Friendica\Util\DateTimeFormat;
-
-class ExpireConversations
-{
- /**
- * Delete old conversation entries
- */
- public static function execute()
- {
- $days = intval(DI::config()->get('system', 'dbclean_expire_conversation', 90));
- if (empty($days)) {
- return;
- }
-
- DBA::delete('conversation', ["`received` < ?", DateTimeFormat::utc('now - ' . $days . ' days')]);
- }
-}
}
Logger::info('Origin item ' . $target_item['id'] . ' with URL ' . $target_item['uri'] . ' will be distributed.');
- } elseif (!DBA::exists('conversation', ['item-uri' => $target_item['uri'], 'protocol' => Conversation::PARCEL_ACTIVITYPUB])) {
+ } elseif (!Post\Activity::exists($target_item['uri-id'])) {
Logger::info('Remote item ' . $target_item['id'] . ' with URL ' . $target_item['uri'] . ' is no AP post. It will not be distributed.');
return ['count' => 0, 'contacts' => []];
} elseif ($parent['origin']) {
use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) {
- define('DB_UPDATE_VERSION', 1474);
+ define('DB_UPDATE_VERSION', 1475);
}
return [
"featured-tags" => ["type" => "varchar(255)", "comment" => "Address for the collection of featured tags"],
"manually-approve" => ["type" => "boolean", "comment" => ""],
"discoverable" => ["type" => "boolean", "comment" => "Mastodon extension: true if profile is published in their directory"],
+ "suspended" => ["type" => "boolean", "comment" => "Mastodon extension: true if profile is suspended"],
"nick" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"name" => ["type" => "varchar(255)", "comment" => ""],
"about" => ["type" => "text", "comment" => ""],
"uid" => ["uid"],
]
],
- "conversation" => [
- "comment" => "Raw data and structure information for messages",
- "fields" => [
- "item-uri" => ["type" => "varbinary(255)", "not null" => "1", "primary" => "1", "comment" => "Original URI of the item - unrelated to the table with the same name"],
- "reply-to-uri" => ["type" => "varbinary(255)", "not null" => "1", "default" => "", "comment" => "URI to which this item is a reply"],
- "conversation-uri" => ["type" => "varbinary(255)", "not null" => "1", "default" => "", "comment" => "GNU Social conversation URI"],
- "conversation-href" => ["type" => "varbinary(255)", "not null" => "1", "default" => "", "comment" => "GNU Social conversation link"],
- "protocol" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "255", "comment" => "The protocol of the item"],
- "direction" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "How the message arrived here: 1=push, 2=pull"],
- "source" => ["type" => "mediumtext", "comment" => "Original source"],
- "received" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Receiving date"],
- ],
- "indexes" => [
- "PRIMARY" => ["item-uri"],
- "conversation-uri" => ["conversation-uri"],
- "received" => ["received"],
- ]
- ],
"workerqueue" => [
"comment" => "Background tasks queue entries",
"fields" => [
"vid" => ["vid"],
]
],
+ "post-activity" => [
+ "comment" => "Original remote activity",
+ "fields" => [
+ "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
+ "activity" => ["type" => "mediumtext", "comment" => "Original activity"],
+ "received" => ["type" => "datetime", "comment" => ""],
+ ],
+ "indexes" => [
+ "PRIMARY" => ["uri-id"],
+ ]
+ ],
"post-category" => [
"comment" => "post relation to categories",
"fields" => [
projects / friendica.git / commitdiff