X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=bluesky%2Fbluesky.php;h=c96658721f8737d020e7c38f8ff35897cf473b48;hb=c89abdb74ce1dc1782393e02a65478edd5716f87;hp=eb8f07a482b673f26a36b0f6c2aef9354356676e;hpb=aa9f0c22816c8c64197031bdcc063c80c16a404d;p=friendica-addons.git diff --git a/bluesky/bluesky.php b/bluesky/bluesky.php index eb8f07a4..c9665872 100644 --- a/bluesky/bluesky.php +++ b/bluesky/bluesky.php @@ -6,25 +6,33 @@ * Author: Michael Vogel * * @todo - * - Links in outgoing comments + * Currently technical issues in the core: * - Outgoing mentions * + * At some point in time: + * - Sending Quote shares https://atproto.com/lexicons/app-bsky-embed#appbskyembedrecord and https://atproto.com/lexicons/app-bsky-embed#appbskyembedrecordwithmedia + * * Possibly not possible: * - only fetch new posts * * Currently not possible, due to limitations in Friendica * - mute contacts https://atproto.com/lexicons/app-bsky-graph#appbskygraphmuteactor * - unmute contacts https://atproto.com/lexicons/app-bsky-graph#appbskygraphunmuteactor + * + * Possibly interesting: + * - https://atproto.com/lexicons/com-atproto-label#comatprotolabelsubscribelabels */ use Friendica\Content\Text\BBCode; use Friendica\Content\Text\HTML; use Friendica\Content\Text\Plaintext; +use Friendica\Core\Cache\Enum\Duration; use Friendica\Core\Config\Util\ConfigFileManager; use Friendica\Core\Hook; use Friendica\Core\Logger; use Friendica\Core\Protocol; use Friendica\Core\Renderer; +use Friendica\Core\System; use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; @@ -35,12 +43,15 @@ use Friendica\Model\Photo; use Friendica\Model\Post; use Friendica\Network\HTTPClient\Client\HttpClientAccept; use Friendica\Network\HTTPClient\Client\HttpClientOptions; +use Friendica\Object\Image; use Friendica\Protocol\Activity; +use Friendica\Protocol\Relay; use Friendica\Util\DateTimeFormat; use Friendica\Util\Strings; const BLUESKY_DEFAULT_POLL_INTERVAL = 10; // given in minutes const BLUESKY_HOST = 'https://bsky.app'; // Hard wired until Bluesky will run on multiple systems +const BLUESKY_IMAGE_SIZE = [1000000, 500000, 100000, 50000]; function bluesky_install() { @@ -110,17 +121,24 @@ function bluesky_probe_detect(array &$hookData) return; } - $data = bluesky_get($pconfig['uid'], '/xrpc/app.bsky.actor.getProfile?actor=' . $did, HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . $token]]]); + $data = bluesky_xrpc_get($pconfig['uid'], 'app.bsky.actor.getProfile', ['actor' => $did]); if (empty($data)) { return; } - $hookData['result'] = bluesky_get_contact($data, 0, $pconfig['uid']); + $hookData['result'] = bluesky_get_contact_fields($data, 0, false); - // Authoritative probe should set the result even if the probe was unsuccessful - if ($hookData['network'] == Protocol::BLUESKY && empty($hookData['result'])) { - $hookData['result'] = []; - } + // Preparing probe data. This differs slightly from the contact array + $hookData['result']['about'] = HTML::toBBCode($data->description ?? ''); + $hookData['result']['photo'] = $data->avatar ?? ''; + $hookData['result']['header'] = $data->banner ?? ''; + $hookData['result']['batch'] = ''; + $hookData['result']['notify'] = ''; + $hookData['result']['poll'] = ''; + $hookData['result']['poco'] = ''; + $hookData['result']['pubkey'] = ''; + $hookData['result']['priority'] = 0; + $hookData['result']['guid'] = ''; } function bluesky_item_by_link(array &$hookData) @@ -148,7 +166,7 @@ function bluesky_item_by_link(array &$hookData) $uri = 'at://' . $did . '/app.bsky.feed.post/' . $matches[2]; - $uri = bluesky_fetch_missing_post($uri, $hookData['uid'], 0); + $uri = bluesky_fetch_missing_post($uri, $hookData['uid'], 0, 0); Logger::debug('Got post', ['profile' => $matches[1], 'cid' => $matches[2], 'result' => $uri]); if (!empty($uri)) { $item = Post::selectFirst(['id'], ['uri' => $uri, 'uid' => $hookData['uid']]); @@ -190,14 +208,14 @@ function bluesky_follow(array &$hook_data) 'createdAt' => DateTimeFormat::utcNow(DateTimeFormat::ATOM), '$type' => 'app.bsky.graph.follow' ]; - + $post = [ 'collection' => 'app.bsky.graph.follow', 'repo' => DI::pConfig()->get($hook_data['uid'], 'bluesky', 'did'), 'record' => $record ]; - - $activity = bluesky_post($hook_data['uid'], '/xrpc/com.atproto.repo.createRecord', json_encode($post), ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . $token]]); + + $activity = bluesky_xrpc_post($hook_data['uid'], 'com.atproto.repo.createRecord', $post); if (!empty($activity->uri)) { $hook_data['contact'] = $contact; Logger::debug('Successfully start following', ['url' => $contact['url'], 'uri' => $activity->uri]); @@ -215,7 +233,7 @@ function bluesky_unfollow(array &$hook_data) return; } - $data = bluesky_get($hook_data['uid'], '/xrpc/app.bsky.actor.getProfile?actor=' . $hook_data['contact']['url'], HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . $token]]]); + $data = bluesky_xrpc_get($hook_data['uid'], 'app.bsky.actor.getProfile', ['actor' => $hook_data['contact']['url']]); if (empty($data->viewer) || empty($data->viewer->following)) { return; } @@ -243,14 +261,14 @@ function bluesky_block(array &$hook_data) 'createdAt' => DateTimeFormat::utcNow(DateTimeFormat::ATOM), '$type' => 'app.bsky.graph.block' ]; - + $post = [ 'collection' => 'app.bsky.graph.block', 'repo' => DI::pConfig()->get($hook_data['uid'], 'bluesky', 'did'), 'record' => $record ]; - - $activity = bluesky_post($hook_data['uid'], '/xrpc/com.atproto.repo.createRecord', json_encode($post), ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . $token]]); + + $activity = bluesky_xrpc_post($hook_data['uid'], 'com.atproto.repo.createRecord', $post); if (!empty($activity->uri)) { $cdata = Contact::getPublicAndUserContactID($hook_data['contact']['id'], $hook_data['uid']); if (!empty($cdata['user'])) { @@ -271,7 +289,7 @@ function bluesky_unblock(array &$hook_data) return; } - $data = bluesky_get($hook_data['uid'], '/xrpc/app.bsky.actor.getProfile?actor=' . $hook_data['contact']['url'], HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . $token]]]); + $data = bluesky_xrpc_get($hook_data['uid'], 'app.bsky.actor.getProfile', ['actor' => $hook_data['contact']['url']]); if (empty($data->viewer) || empty($data->viewer->blocking)) { return; } @@ -287,26 +305,28 @@ function bluesky_settings(array &$data) return; } - $enabled = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'post') ?? false; - $def_enabled = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'post_by_default') ?? false; - $host = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'host') ?: 'https://bsky.social'; - $handle = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'handle'); - $did = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'did'); - $token = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'access_token'); - $import = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'import') ?? false; + $enabled = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'post') ?? false; + $def_enabled = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'post_by_default') ?? false; + $host = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'host') ?: 'https://bsky.social'; + $handle = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'handle'); + $did = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'did'); + $token = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'access_token'); + $import = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'import') ?? false; + $import_feeds = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'import_feeds') ?? false; $status = $token ? DI::l10n()->t("You are authenticated to Bluesky. For security reasons the password isn't stored.") : DI::l10n()->t('You are not authenticated. Please enter the app password.'); $t = Renderer::getMarkupTemplate('connector_settings.tpl', 'addon/bluesky/'); $html = Renderer::replaceMacros($t, [ - '$enable' => ['bluesky', DI::l10n()->t('Enable Bluesky Post Addon'), $enabled], - '$bydefault' => ['bluesky_bydefault', DI::l10n()->t('Post to Bluesky by default'), $def_enabled], - '$import' => ['bluesky_import', DI::l10n()->t('Import the remote timeline'), $import], - '$host' => ['bluesky_host', DI::l10n()->t('Bluesky host'), $host, '', '', 'readonly'], - '$handle' => ['bluesky_handle', DI::l10n()->t('Bluesky handle'), $handle], - '$did' => ['bluesky_did', DI::l10n()->t('Bluesky DID'), $did, DI::l10n()->t('This is the unique identifier. It will be fetched automatically, when the handle is entered.'), '', 'readonly'], - '$password' => ['bluesky_password', DI::l10n()->t('Bluesky app password'), '', DI::l10n()->t("Please don't add your real password here, but instead create a specific app password in the Bluesky settings.")], - '$status' => $status + '$enable' => ['bluesky', DI::l10n()->t('Enable Bluesky Post Addon'), $enabled], + '$bydefault' => ['bluesky_bydefault', DI::l10n()->t('Post to Bluesky by default'), $def_enabled], + '$import' => ['bluesky_import', DI::l10n()->t('Import the remote timeline'), $import], + '$import_feeds' => ['bluesky_import_feeds', DI::l10n()->t('Import the pinned feeds'), $import_feeds, DI::l10n()->t('When activated, Posts will be imported from all the feeds that you pinned in Bluesky.')], + '$host' => ['bluesky_host', DI::l10n()->t('Bluesky host'), $host, '', '', 'readonly'], + '$handle' => ['bluesky_handle', DI::l10n()->t('Bluesky handle'), $handle], + '$did' => ['bluesky_did', DI::l10n()->t('Bluesky DID'), $did, DI::l10n()->t('This is the unique identifier. It will be fetched automatically, when the handle is entered.'), '', 'readonly'], + '$password' => ['bluesky_password', DI::l10n()->t('Bluesky app password'), '', DI::l10n()->t("Please don't add your real password here, but instead create a specific app password in the Bluesky settings.")], + '$status' => $status ]); $data = [ @@ -336,6 +356,7 @@ function bluesky_settings_post(array &$b) DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'host', $host); DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'handle', $handle); DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'import', intval($_POST['bluesky_import'])); + DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'import_feeds', intval($_POST['bluesky_import_feeds'])); if (!empty($host) && !empty($handle)) { if (empty($old_did) || $old_host != $host || $old_handle != $handle) { @@ -402,12 +423,18 @@ function bluesky_cron() } } - Logger::notice('importing timeline - start', ['user' => $pconfig['uid']]); - bluesky_fetch_timeline($pconfig['uid']); - Logger::notice('importing timeline - done', ['user' => $pconfig['uid']]); - Logger::notice('importing notifications - start', ['user' => $pconfig['uid']]); - bluesky_fetch_notifications($pconfig['uid']); - Logger::notice('importing notifications - done', ['user' => $pconfig['uid']]); + // Refresh the token now, so that it doesn't need to be refreshed in parallel by the following workers + bluesky_get_token($pconfig['uid']); + + Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'addon/bluesky/bluesky_timeline.php', $pconfig['uid']); + Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'addon/bluesky/bluesky_notifications.php', $pconfig['uid']); + + if (DI::pConfig()->get($pconfig['uid'], 'bluesky', 'import_feeds')) { + $feeds = bluesky_get_feeds($pconfig['uid']); + foreach ($feeds as $feed) { + Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'addon/bluesky/bluesky_feed.php', $pconfig['uid'], $feed); + } + } } $last_clean = DI::keyValue()->get('bluesky_last_clean'); @@ -566,7 +593,7 @@ function bluesky_create_activity(array $item, stdClass $parent = null) ]; } - $activity = bluesky_post($uid, '/xrpc/com.atproto.repo.createRecord', json_encode($post), ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . $token]]); + $activity = bluesky_xrpc_post($uid, 'com.atproto.repo.createRecord', $post); if (empty($activity)) { return; } @@ -585,12 +612,13 @@ function bluesky_create_post(array $item, stdClass $root = null, stdClass $paren } $did = DI::pConfig()->get($uid, 'bluesky', 'did'); - $urls = bluesky_get_urls($item['body']); + $urls = bluesky_get_urls(Post\Media::removeFromBody($item['body'])); + $item['body'] = $urls['body']; - $msg = Plaintext::getPost($item, 300, false, BBCode::CONNECTORS); + $msg = Plaintext::getPost($item, 300, false, BBCode::BLUESKY); foreach ($msg['parts'] as $key => $part) { - $facets = bluesky_get_facets($part, $urls); + $facets = bluesky_get_facets($part, $urls['urls']); $record = [ 'text' => $facets['body'], @@ -608,6 +636,12 @@ function bluesky_create_post(array $item, stdClass $root = null, stdClass $paren if ($key == count($msg['parts']) - 1) { $record = bluesky_add_embed($uid, $msg, $record); + if (empty($record)) { + if (Worker::getRetrial() < 3) { + Worker::defer(); + } + return; + } } $post = [ @@ -616,8 +650,11 @@ function bluesky_create_post(array $item, stdClass $root = null, stdClass $paren 'record' => $record ]; - $parent = bluesky_post($uid, '/xrpc/com.atproto.repo.createRecord', json_encode($post), ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . $token]]); + $parent = bluesky_xrpc_post($uid, 'com.atproto.repo.createRecord', $post); if (empty($parent)) { + if ($part == 0) { + Worker::defer(); + } return; } Logger::debug('Posting done', ['return' => $parent]); @@ -634,25 +671,52 @@ function bluesky_create_post(array $item, stdClass $root = null, stdClass $paren function bluesky_get_urls(string $body): array { - // Remove all hashtags and mentions - $body = preg_replace("/([#@!])\[url\=(.*?)\](.*?)\[\/url\]/ism", '', $body); + // Remove all hashtag and mention links + $body = preg_replace("/([#@!])\[url\=(.*?)\](.*?)\[\/url\]/ism", '$1$3', $body); + $body = BBCode::expandVideoLinks($body); $urls = []; // Search for pure links - if (preg_match_all("/\[url\](https?:.*?)\[\/url\]/ism", $body, $matches)) { - foreach ($matches[1] as $url) { - $urls[] = $url; + if (preg_match_all("/\[url\](https?:.*?)\[\/url\]/ism", $body, $matches, PREG_SET_ORDER)) { + foreach ($matches as $match) { + $text = Strings::getStyledURL($match[1]); + $hash = bluesky_get_hash_for_url($match[0], mb_strlen($text)); + $urls[] = ['url' => $match[1], 'text' => $text, 'hash' => $hash]; + $body = str_replace($match[0], $hash, $body); } } // Search for links with descriptions - if (preg_match_all("/\[url\=(https?:.*?)\].*?\[\/url\]/ism", $body, $matches)) { - foreach ($matches[1] as $url) { - $urls[] = $url; + if (preg_match_all("/\[url\=(https?:.*?)\](.*?)\[\/url\]/ism", $body, $matches, PREG_SET_ORDER)) { + foreach ($matches as $match) { + if ($match[1] == $match[2]) { + $text = Strings::getStyledURL($match[1]); + } else { + $text = $match[2]; + } + if (mb_strlen($text) < 100) { + $hash = bluesky_get_hash_for_url($match[0], mb_strlen($text)); + $urls[] = ['url' => $match[1], 'text' => $text, 'hash' => $hash]; + $body = str_replace($match[0], $hash, $body); + } else { + $text = Strings::getStyledURL($match[1]); + $hash = bluesky_get_hash_for_url($match[0], mb_strlen($text)); + $urls[] = ['url' => $match[1], 'text' => $text, 'hash' => $hash]; + $body = str_replace($match[0], $text . ' ' . $hash, $body); + } } } - return $urls; + + return ['body' => $body, 'urls' => $urls]; +} + +function bluesky_get_hash_for_url(string $text, int $linklength): string +{ + if ($linklength <= 10) { + return '|' . hash('crc32', $text) . '|'; + } + return substr('|' . hash('crc32', $text) . base64_encode($text), 0, $linklength - 2) . '|'; } function bluesky_get_facets(string $body, array $urls): array @@ -660,7 +724,7 @@ function bluesky_get_facets(string $body, array $urls): array $facets = []; foreach ($urls as $url) { - $pos = strpos($body, $url); + $pos = strpos($body, $url['hash']); if ($pos === false) { continue; } @@ -669,16 +733,16 @@ function bluesky_get_facets(string $body, array $urls): array } else { $prefix = ''; } - $linktext = Strings::getStyledURL($url); - $body = $prefix . $linktext . substr($body, $pos + strlen($url)); + + $body = $prefix . $url['text'] . substr($body, $pos + strlen($url['hash'])); $facet = new stdClass; $facet->index = new stdClass; - $facet->index->byteEnd = $pos + strlen($linktext); + $facet->index->byteEnd = $pos + strlen($url['text']); $facet->index->byteStart = $pos; $feature = new stdClass; - $feature->uri = $url; + $feature->uri = $url['url']; $type = '$type'; $feature->$type = 'app.bsky.richtext.facet#link'; @@ -694,12 +758,15 @@ function bluesky_add_embed(int $uid, array $msg, array $record): array if (($msg['type'] != 'link') && !empty($msg['images'])) { $images = []; foreach ($msg['images'] as $image) { - $photo = Photo::selectFirst(['resource-id'], ['id' => $image['id']]); - $photo = Photo::selectFirst([], ["`resource-id` = ? AND `scale` > ?", $photo['resource-id'], 0], ['order' => ['scale']]); + if (count($images) == 4) { + continue; + } + $photo = Photo::selectFirst([], ['id' => $image['id']]); $blob = bluesky_upload_blob($uid, $photo); - if (!empty($blob) && count($images) < 4) { - $images[] = ['alt' => $image['description'] ?? '', 'image' => $blob]; + if (empty($blob)) { + return []; } + $images[] = ['alt' => $image['description'] ?? '', 'image' => $blob]; } if (!empty($images)) { $record['embed'] = ['$type' => 'app.bsky.embed.images', 'images' => $images]; @@ -709,8 +776,8 @@ function bluesky_add_embed(int $uid, array $msg, array $record): array '$type' => 'app.bsky.embed.external', 'external' => [ 'uri' => $msg['url'], - 'title' => $msg['title'], - 'description' => $msg['description'], + 'title' => $msg['title'] ?? '', + 'description' => $msg['description'] ?? '', ] ]; if (!empty($msg['image'])) { @@ -726,31 +793,46 @@ function bluesky_add_embed(int $uid, array $msg, array $record): array function bluesky_upload_blob(int $uid, array $photo): ?stdClass { + $retrial = Worker::getRetrial(); $content = Photo::getImageForPhoto($photo); + + $picture = new Image($content, $photo['type']); + $height = $picture->getHeight(); + $width = $picture->getWidth(); + $size = strlen($content); + + $picture = Photo::resizeToFileSize($picture, BLUESKY_IMAGE_SIZE[$retrial]); + $new_height = $picture->getHeight(); + $new_width = $picture->getWidth(); + $content = $picture->asString(); + $new_size = strlen($content); + + Logger::info('Uploading', ['uid' => $uid, 'retrial' => $retrial, 'height' => $new_height, 'width' => $new_width, 'size' => $new_size, 'orig-height' => $height, 'orig-width' => $width, 'orig-size' => $size]); + $data = bluesky_post($uid, '/xrpc/com.atproto.repo.uploadBlob', $content, ['Content-type' => $photo['type'], 'Authorization' => ['Bearer ' . bluesky_get_token($uid)]]); if (empty($data)) { + Logger::info('Uploading failed', ['uid' => $uid, 'retrial' => $retrial, 'height' => $new_height, 'width' => $new_width, 'size' => $new_size, 'orig-height' => $height, 'orig-width' => $width, 'orig-size' => $size]); return null; } - Logger::debug('Uploaded blob', ['return' => $data]); + Logger::debug('Uploaded blob', ['return' => $data, 'uid' => $uid, 'retrial' => $retrial, 'height' => $new_height, 'width' => $new_width, 'size' => $new_size, 'orig-height' => $height, 'orig-width' => $width, 'orig-size' => $size]); return $data->blob; } function bluesky_delete_post(string $uri, int $uid) { - $token = bluesky_get_token($uid); $parts = bluesky_get_uri_parts($uri); if (empty($parts)) { Logger::debug('No uri delected', ['uri' => $uri]); return; } - bluesky_post($uid, '/xrpc/com.atproto.repo.deleteRecord', json_encode($parts), ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . $token]]); + bluesky_xrpc_post($uid, 'com.atproto.repo.deleteRecord', $parts); Logger::debug('Deleted', ['parts' => $parts]); } function bluesky_fetch_timeline(int $uid) { - $data = bluesky_get($uid, '/xrpc/app.bsky.feed.getTimeline', HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . bluesky_get_token($uid)]]]); + $data = bluesky_xrpc_get($uid, 'app.bsky.feed.getTimeline'); if (empty($data)) { return; } @@ -760,7 +842,7 @@ function bluesky_fetch_timeline(int $uid) } foreach (array_reverse($data->feed) as $entry) { - bluesky_process_post($entry->post, $uid); + bluesky_process_post($entry->post, $uid, Item::PR_NONE, 0); if (!empty($entry->reason)) { bluesky_process_reason($entry->reason, bluesky_get_uri($entry->post), $uid); } @@ -801,6 +883,7 @@ function bluesky_process_reason(stdClass $reason, string $uri, int $uid) return; } + $item['guid'] = Item::guidFromUri($item['uri'], $contact['alias']); $item['owner-name'] = $item['author-name']; $item['owner-link'] = $item['author-link']; $item['owner-avatar'] = $item['author-avatar']; @@ -812,11 +895,11 @@ function bluesky_process_reason(stdClass $reason, string $uri, int $uid) function bluesky_fetch_notifications(int $uid) { - $result = bluesky_get($uid, '/xrpc/app.bsky.notification.listNotifications', HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . bluesky_get_token($uid)]]]); - if (empty($result->notifications)) { + $data = bluesky_xrpc_get($uid, 'app.bsky.notification.listNotifications'); + if (empty($data->notifications)) { return; } - foreach ($result->notifications as $notification) { + foreach ($data->notifications as $notification) { $uri = bluesky_get_uri($notification); if (Post::exists(['uri' => $uri, 'uid' => $uid]) || Post::exists(['extid' => $uri, 'uid' => $uid])) { Logger::debug('Notification already processed', ['uid' => $uid, 'reason' => $notification->reason, 'uri' => $uri, 'indexedAt' => $notification->indexedAt]); @@ -829,47 +912,80 @@ function bluesky_fetch_notifications(int $uid) $item['gravity'] = Item::GRAVITY_ACTIVITY; $item['body'] = $item['verb'] = Activity::LIKE; $item['thr-parent'] = bluesky_get_uri($notification->record->subject); - $result = Item::insert($item); - Logger::debug('Got like', ['uid' => $uid, 'result' => $result]); - break; + $item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $item['contact-id'], 0); + if (!empty($item['thr-parent'])) { + $data = Item::insert($item); + Logger::debug('Got like', ['uid' => $uid, 'result' => $data, 'uri' => $uri]); + } else { + Logger::info('Thread parent not found', ['uid' => $uid, 'parent' => $item['thr-parent'], 'uri' => $uri]); + } + break; case 'repost': $item = bluesky_get_header($notification, $uri, $uid, $uid); $item['gravity'] = Item::GRAVITY_ACTIVITY; $item['body'] = $item['verb'] = Activity::ANNOUNCE; $item['thr-parent'] = bluesky_get_uri($notification->record->subject); - $result = Item::insert($item); - Logger::debug('Got repost', ['uid' => $uid, 'result' => $result]); - break; + $item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $item['contact-id'], 0); + if (!empty($item['thr-parent'])) { + $data = Item::insert($item); + Logger::debug('Got repost', ['uid' => $uid, 'result' => $data, 'uri' => $uri]); + } else { + Logger::info('Thread parent not found', ['uid' => $uid, 'parent' => $$item['thr-parent'], 'uri' => $uri]); + } + break; case 'follow': $contact = bluesky_get_contact($notification->author, $uid, $uid); - Logger::debug('New follower', ['uid' => $uid, 'nick' => $contact['nick']]); + Logger::debug('New follower', ['uid' => $uid, 'nick' => $contact['nick'], 'uri' => $uri]); break; case 'mention': - $result = bluesky_process_post($notification, $uid); - Logger::debug('Got mention', ['uid' => $uid, 'result' => $result]); + $data = bluesky_process_post($notification, $uid, Item::PR_PUSHED, 0); + Logger::debug('Got mention', ['uid' => $uid, 'result' => $data, 'uri' => $uri]); break; case 'reply': - $result = bluesky_process_post($notification, $uid); - Logger::debug('Got reply', ['uid' => $uid, 'result' => $result]); + $data = bluesky_process_post($notification, $uid, Item::PR_PUSHED, 0); + Logger::debug('Got reply', ['uid' => $uid, 'result' => $data, 'uri' => $uri]); break; case 'quote': - $result = bluesky_process_post($notification, $uid); - Logger::debug('Got quote', ['uid' => $uid, 'result' => $result]); + $data = bluesky_process_post($notification, $uid, Item::PR_PUSHED, 0); + Logger::debug('Got quote', ['uid' => $uid, 'result' => $data, 'uri' => $uri]); break; default: - Logger::notice('Unhandled reason', ['reason' => $notification->reason]); + Logger::notice('Unhandled reason', ['reason' => $notification->reason, 'uri' => $uri]); break; } } } -function bluesky_process_post(stdClass $post, int $uid): int +function bluesky_fetch_feed(int $uid, string $feed) +{ + $data = bluesky_xrpc_get($uid, 'app.bsky.feed.getFeed', ['feed' => $feed]); + if (empty($data)) { + return; + } + + if (empty($data->feed)) { + return; + } + + foreach (array_reverse($data->feed) as $entry) { + if (!Relay::isWantedLanguage($entry->post->record->text)) { + Logger::debug('Unwanted language detected', ['text' => $entry->post->record->text]); + continue; + } + bluesky_process_post($entry->post, $uid, Item::PR_TAG, 0); + if (!empty($entry->reason)) { + bluesky_process_reason($entry->reason, bluesky_get_uri($entry->post), $uid); + } + } +} + +function bluesky_process_post(stdClass $post, int $uid, int $post_reason, $level): int { $uri = bluesky_get_uri($post); @@ -877,15 +993,22 @@ function bluesky_process_post(stdClass $post, int $uid): int return 0; } - Logger::debug('Importing post', ['uid' => $uid, 'indexedAt' => $post->indexedAt, 'uri' => $post->uri, 'cid' => $post->cid]); + Logger::debug('Importing post', ['uid' => $uid, 'indexedAt' => $post->indexedAt, 'uri' => $post->uri, 'cid' => $post->cid, 'root' => $post->record->reply->root ?? '']); $item = bluesky_get_header($post, $uri, $uid, $uid); - - $item = bluesky_get_content($item, $post->record, $uid); + $item = bluesky_get_content($item, $post->record, $uri, $uid, $level); + if (empty($item)) { + return 0; + } if (!empty($post->embed)) { - $item = bluesky_add_media($post->embed, $item, $uid); + $item = bluesky_add_media($post->embed, $item, $uid, $level); + } + + if (empty($item['post-reason'])) { + $item['post-reason'] = $post_reason; } + return item::insert($item); } @@ -916,24 +1039,43 @@ function bluesky_get_header(stdClass $post, string $uri, int $uid, int $fetch_ui $item['owner-link'] = $item['author-link']; $item['owner-avatar'] = $item['author-avatar']; + if (in_array($contact['rel'], [Contact::SHARING, Contact::FRIEND])) { + $item['post-reason'] = Item::PR_FOLLOWER; + } + return $item; } -function bluesky_get_content(array $item, stdClass $record, int $uid): array +function bluesky_get_content(array $item, stdClass $record, string $uri, int $uid, int $level): array { + if (empty($item)) { + return []; + } + if (!empty($record->reply)) { $item['parent-uri'] = bluesky_get_uri($record->reply->root); - $item['parent-uri'] = bluesky_fetch_missing_post($item['parent-uri'], $uid, $item['contact-id']); + if ($item['parent-uri'] != $uri) { + $item['parent-uri'] = bluesky_fetch_missing_post($item['parent-uri'], $uid, $item['contact-id'], $level); + if (empty($item['parent-uri'])) { + return []; + } + } + $item['thr-parent'] = bluesky_get_uri($record->reply->parent); - $item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $item['contact-id']); + if (!in_array($item['thr-parent'], [$uri, $item['parent-uri']])) { + $item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $item['contact-id'], $level, $item['parent-uri']); + if (empty($item['thr-parent'])) { + return []; + } + } } - $item['body'] = bluesky_get_text($record, $uid); + $item['body'] = bluesky_get_text($record); $item['created'] = DateTimeFormat::utc($record->createdAt, DateTimeFormat::MYSQL); return $item; } -function bluesky_get_text(stdClass $record, int $uid): string +function bluesky_get_text(stdClass $record): string { $text = $record->text; @@ -962,7 +1104,7 @@ function bluesky_get_text(stdClass $record, int $uid): string break; case 'app.bsky.richtext.facet#mention': - $contact = Contact::selectFirst(['id'], ['nurl' => $feature->did, 'uid' => [0, $uid]]); + $contact = Contact::getByURL($feature->did, null, ['id']); if (!empty($contact['id'])) { $url = DI::baseUrl() . '/contact/' . $contact['id']; if (substr($linktext, 0, 1) == '@') { @@ -984,7 +1126,7 @@ function bluesky_get_text(stdClass $record, int $uid): string return $text; } -function bluesky_add_media(stdClass $embed, array $item, int $fetch_uid): array +function bluesky_add_media(stdClass $embed, array $item, int $fetch_uid, int $level): array { $type = '$type'; switch ($embed->$type) { @@ -1016,13 +1158,16 @@ function bluesky_add_media(stdClass $embed, array $item, int $fetch_uid): array $uri = bluesky_get_uri($embed->record); $shared = Post::selectFirst(['uri-id'], ['uri' => $uri, 'uid' => $item['uid']]); if (empty($shared)) { + if (empty($embed->record->value)) { + Logger::info('Record has got no value', ['record' => $embed->record]); + break; + } $shared = bluesky_get_header($embed->record, $uri, 0, $fetch_uid); + $shared = bluesky_get_content($shared, $embed->record->value, $uri, $item['uid'], $level); if (!empty($shared)) { - $shared = bluesky_get_content($shared, $embed->record->value, $item['uid']); - if (!empty($embed->record->embeds)) { foreach ($embed->record->embeds as $single) { - $shared = bluesky_add_media($single, $shared, $fetch_uid); + $shared = bluesky_add_media($single, $shared, $fetch_uid, $level); } } $id = Item::insert($shared); @@ -1039,19 +1184,14 @@ function bluesky_add_media(stdClass $embed, array $item, int $fetch_uid): array $shared = Post::selectFirst(['uri-id'], ['uri' => $uri, 'uid' => $item['uid']]); if (empty($shared)) { $shared = bluesky_get_header($embed->record->record, $uri, 0, $fetch_uid); + $shared = bluesky_get_content($shared, $embed->record->record->value, $uri, $item['uid'], $level); if (!empty($shared)) { - $shared = bluesky_get_content($shared, $embed->record->record->value, $item['uid']); - - if (!empty($embed->record->embeds)) { + if (!empty($embed->record->record->embeds)) { foreach ($embed->record->record->embeds as $single) { - $shared = bluesky_add_media($single, $shared, $fetch_uid); + $shared = bluesky_add_media($single, $shared, $fetch_uid, $level); } } - if (!empty($embed->media)) { - bluesky_add_media($embed->media, $item, $fetch_uid); - } - $id = Item::insert($shared); $shared = Post::selectFirst(['uri-id'], ['id' => $id]); } @@ -1059,6 +1199,10 @@ function bluesky_add_media(stdClass $embed, array $item, int $fetch_uid): array if (!empty($shared)) { $item['quote-uri-id'] = $shared['uri-id']; } + + if (!empty($embed->media)) { + bluesky_add_media($embed->media, $item, $fetch_uid, $level); + } break; default: @@ -1070,6 +1214,10 @@ function bluesky_add_media(stdClass $embed, array $item, int $fetch_uid): array function bluesky_get_uri(stdClass $post): string { + if (empty($post->cid)) { + Logger::info('Invalid URI', ['post' => $post, 'callstack' => System::callstack(10, 0, true)]); + return ''; + } return $post->uri . ':' . $post->cid; } @@ -1116,47 +1264,92 @@ function bluesky_get_uri_parts(string $uri): ?stdClass return $class; } -function bluesky_fetch_missing_post(string $uri, int $uid, int $causer): string +function bluesky_fetch_missing_post(string $uri, int $uid, int $causer, int $level, string $fallback = ''): string { - if (Post::exists(['uri' => $uri, 'uid' => [$uid, 0]])) { - Logger::debug('Post exists', ['uri' => $uri]); - return $uri; + $fetched_uri = bluesky_fetch_post($uri, $uid); + if (!empty($fetched_uri)) { + return $fetched_uri; } - $reply = Post::selectFirst(['uri'], ['extid' => $uri, 'uid' => [$uid, 0]]); - if (!empty($reply['uri'])) { - return $reply['uri']; + if (++$level > 100) { + Logger::info('Recursion level too deep', ['level' => $level, 'uid' => $uid, 'uri' => $uri, 'fallback' => $fallback]); + // When the level is too deep we will fallback to the parent uri. + // Allthough the threading won't be correct, we at least had stored all posts and won't try again + return $fallback; } - Logger::debug('Fetch missing post', ['uri' => $uri]); $class = bluesky_get_uri_class($uri); $fetch_uri = $class->uri; - $data = bluesky_get($uid, '/xrpc/app.bsky.feed.getPosts?uris=' . urlencode($fetch_uri), HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . bluesky_get_token($uid)]]]); + Logger::debug('Fetch missing post', ['level' => $level, 'uid' => $uid, 'uri' => $uri]); + $data = bluesky_xrpc_get($uid, 'app.bsky.feed.getPostThread', ['uri' => $fetch_uri]); if (empty($data)) { - return ''; + Logger::info('Thread was not fetched', ['level' => $level, 'uid' => $uid, 'uri' => $uri, 'fallback' => $fallback]); + return $fallback; } + Logger::debug('Reply count', ['replies' => $data->thread->post->replyCount, 'level' => $level, 'uid' => $uid, 'uri' => $uri]); + if ($causer != 0) { $cdata = Contact::getPublicAndUserContactID($causer, $uid); + } else { + $cdata = []; } - foreach ($data->posts as $post) { - $uri = bluesky_get_uri($post); - $item = bluesky_get_header($post, $uri, $uid, $uid); - $item = bluesky_get_content($item, $post->record, $uid); + return bluesky_process_thread($data->thread, $uid, $cdata, $level); +} - $item['post-reason'] = Item::PR_FETCHED; +function bluesky_fetch_post(string $uri, int $uid): string +{ + if (Post::exists(['uri' => $uri, 'uid' => [$uid, 0]])) { + Logger::debug('Post exists', ['uri' => $uri]); + return $uri; + } - if (!empty($cdata['public'])) { - $item['causer-id'] = $cdata['public']; - } + $reply = Post::selectFirst(['uri'], ['extid' => $uri, 'uid' => [$uid, 0]]); + if (!empty($reply['uri'])) { + Logger::debug('Post with extid exists', ['uri' => $uri]); + return $reply['uri']; + } + return ''; +} + +function bluesky_process_thread(stdClass $thread, int $uid, array $cdata, int $level): string +{ + $uri = bluesky_get_uri($thread->post); + $fetched_uri = bluesky_fetch_post($uri, $uid); + if (empty($fetched_uri)) { + Logger::debug('Process missing post', ['uri' => $uri]); + $item = bluesky_get_header($thread->post, $uri, $uid, $uid); + $item = bluesky_get_content($item, $thread->post->record, $uri, $uid, $level); + if (!empty($item)) { + $item['post-reason'] = Item::PR_FETCHED; + + if (!empty($cdata['public'])) { + $item['causer-id'] = $cdata['public']; + } - if (!empty($post->embed)) { - $item = bluesky_add_media($post->embed, $item, $uid); + if (!empty($thread->post->embed)) { + $item = bluesky_add_media($thread->post->embed, $item, $uid, $level); + } + $id = Item::insert($item); + if (!$id) { + Logger::info('Item has not not been stored', ['uri' => $uri]); + return ''; + } + Logger::debug('Stored item', ['id' => $id, 'uri' => $uri]); + } else { + Logger::info('Post has not not been fetched', ['uri' => $uri]); + return ''; } - $id = Item::insert($item); - Logger::debug('Stored item', ['id' => $id, 'uri' => $uri]); + } else { + Logger::debug('Post exists', ['uri' => $uri]); + $uri = $fetched_uri; + } + + foreach ($thread->replies ?? [] as $reply) { + $reply_uri = bluesky_process_thread($reply, $uid, $cdata, $level); + Logger::debug('Reply has been processed', ['uri' => $uri, 'reply' => $reply_uri]); } return $uri; @@ -1220,10 +1413,11 @@ function bluesky_get_contact_fields(stdClass $author, int $uid, bool $update): a 'blocked' => false, 'readonly' => false, 'pending' => false, + 'baseurl' => BLUESKY_HOST, 'url' => $author->did, 'nurl' => $author->did, 'alias' => BLUESKY_HOST . '/profile/' . $author->handle, - 'name' => $author->displayName, + 'name' => $author->displayName ?? $author->handle, 'nick' => $author->handle, 'addr' => $author->handle, ]; @@ -1233,7 +1427,7 @@ function bluesky_get_contact_fields(stdClass $author, int $uid, bool $update): a return $fields; } - $data = bluesky_get($uid, '/xrpc/app.bsky.actor.getProfile?actor=' . $author->did, HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . bluesky_get_token($uid)]]]); + $data = bluesky_xrpc_get($uid, 'app.bsky.actor.getProfile', ['actor' => $author->did]); if (empty($data)) { Logger::debug('Error fetching contact fields', ['uid' => $uid, 'url' => $fields['url']]); return $fields; @@ -1265,9 +1459,35 @@ function bluesky_get_contact_fields(stdClass $author, int $uid, bool $update): a return $fields; } +function bluesky_get_feeds(int $uid): array +{ + $type = '$type'; + $preferences = bluesky_get_preferences($uid); + foreach ($preferences->preferences as $preference) { + if ($preference->$type == 'app.bsky.actor.defs#savedFeedsPref') { + return $preference->pinned ?? []; + } + } + return []; +} + +function bluesky_get_preferences(int $uid): stdClass +{ + $cachekey = 'bluesky:preferences:' . $uid; + $data = DI::cache()->get($cachekey); + if (!is_null($data)) { + return $data; + } + + $data = bluesky_xrpc_get($uid, 'app.bsky.actor.getPreferences'); + + DI::cache()->set($cachekey, $data, Duration::HOUR); + return $data; +} + function bluesky_get_did(int $uid, string $handle): string { - $data = bluesky_get($uid, '/xrpc/com.atproto.identity.resolveHandle?handle=' . $handle); + $data = bluesky_get($uid, '/xrpc/com.atproto.identity.resolveHandle?handle=' . urlencode($handle)); if (empty($data)) { return ''; } @@ -1321,6 +1541,11 @@ function bluesky_create_token(int $uid, string $password): string return $data->accessJwt; } +function bluesky_xrpc_post(int $uid, string $url, $parameters): ?stdClass +{ + return bluesky_post($uid, '/xrpc/' . $url, json_encode($parameters), ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . bluesky_get_token($uid)]]); +} + function bluesky_post(int $uid, string $url, string $params, array $headers): ?stdClass { try { @@ -1338,6 +1563,15 @@ function bluesky_post(int $uid, string $url, string $params, array $headers): ?s return json_decode($curlResult->getBody()); } +function bluesky_xrpc_get(int $uid, string $url, array $parameters = []): ?stdClass +{ + if (!empty($parameters)) { + $url .= '?' . http_build_query($parameters); + } + + return bluesky_get($uid, '/xrpc/' . $url, HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . bluesky_get_token($uid)]]]); +} + function bluesky_get(int $uid, string $url, string $accept_content = HttpClientAccept::DEFAULT, array $opts = []): ?stdClass { try { @@ -1353,4 +1587,4 @@ function bluesky_get(int $uid, string $url, string $accept_content = HttpClientA } return json_decode($curlResult->getBody()); -} \ No newline at end of file +}