X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=bluesky%2Fbluesky.php;h=dac95437e24d52ecab5de11e1a55e116bbc557c4;hb=82073db292d8f5bacdbffab8b94347f3bae1b271;hp=dd56f9a6034d4d4243a9f3bc7066defd8d4b029b;hpb=5c45c05cdb0d5898f624b626b12413dc47fdde96;p=friendica-addons.git diff --git a/bluesky/bluesky.php b/bluesky/bluesky.php index dd56f9a6..dac95437 100644 --- a/bluesky/bluesky.php +++ b/bluesky/bluesky.php @@ -6,46 +6,67 @@ * Author: Michael Vogel * * @todo - * Nice to have: - * - Probing for contacts + * Currently technical issues in the core: + * - Outgoing mentions * - * Need more information: + * At some point in time: + * - Sending Quote shares https://atproto.com/lexicons/app-bsky-embed#appbskyembedrecord and https://atproto.com/lexicons/app-bsky-embed#appbskyembedrecordwithmedia + * + * Possibly not possible: * - only fetch new posts - * - detect contact relations - * - receive likes - * - follow contacts - * - unfollow contacts * - * Possible but less important: - * - Block contacts - * - unblock contacts - * - mute contacts - * - unmute contacts + * Currently not possible, due to limitations in Friendica + * - mute contacts https://atproto.com/lexicons/app-bsky-graph#appbskygraphmuteactor + * - unmute contacts https://atproto.com/lexicons/app-bsky-graph#appbskygraphunmuteactor + * + * Possibly interesting: + * - https://atproto.com/lexicons/com-atproto-label#comatprotolabelsubscribelabels */ use Friendica\Content\Text\BBCode; use Friendica\Content\Text\HTML; use Friendica\Content\Text\Plaintext; +use Friendica\Core\Cache\Enum\Duration; use Friendica\Core\Config\Util\ConfigFileManager; use Friendica\Core\Hook; use Friendica\Core\Logger; use Friendica\Core\Protocol; use Friendica\Core\Renderer; +use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; +use Friendica\Model\GServer; use Friendica\Model\Item; use Friendica\Model\ItemURI; use Friendica\Model\Photo; use Friendica\Model\Post; +use Friendica\Model\Tag; use Friendica\Network\HTTPClient\Client\HttpClientAccept; use Friendica\Network\HTTPClient\Client\HttpClientOptions; +use Friendica\Object\Image; use Friendica\Protocol\Activity; +use Friendica\Protocol\Relay; use Friendica\Util\DateTimeFormat; use Friendica\Util\Strings; const BLUESKY_DEFAULT_POLL_INTERVAL = 10; // given in minutes -const BLUESKY_HOST = 'https://bsky.app'; // Hard wired until Bluesky will run on multiple systems +const BLUESKY_IMAGE_SIZE = [1000000, 500000, 100000, 50000]; + +const BLUEKSY_STATUS_UNKNOWN = 0; +const BLUEKSY_STATUS_TOKEN_OK = 1; +const BLUEKSY_STATUS_SUCCESS = 2; +const BLUEKSY_STATUS_API_FAIL = 10; +const BLUEKSY_STATUS_DID_FAIL = 11; +const BLUEKSY_STATUS_PDS_FAIL = 12; +const BLUEKSY_STATUS_TOKEN_FAIL = 13; + +/* + * (Currently) hard wired paths for Bluesky services + */ +const BLUESKY_DIRECTORY = 'https://plc.directory'; // Path to the directory server service to fetch the PDS of a given DID +const BLUESKY_PDS = 'https://bsky.social'; // Path to the personal data server service (PDS) to fetch the DID for a given handle +const BLUESKY_WEB = 'https://bsky.app'; // Path to the web interface with the user profile and posts function bluesky_install() { @@ -57,14 +78,14 @@ function bluesky_install() Hook::register('connector_settings', __FILE__, 'bluesky_settings'); Hook::register('connector_settings_post', __FILE__, 'bluesky_settings_post'); Hook::register('cron', __FILE__, 'bluesky_cron'); - // Hook::register('support_follow', __FILE__, 'bluesky_support_follow'); - // Hook::register('support_probe', __FILE__, 'bluesky_support_probe'); - // Hook::register('follow', __FILE__, 'bluesky_follow'); - // Hook::register('unfollow', __FILE__, 'bluesky_unfollow'); - // Hook::register('block', __FILE__, 'bluesky_block'); - // Hook::register('unblock', __FILE__, 'bluesky_unblock'); + Hook::register('support_follow', __FILE__, 'bluesky_support_follow'); + Hook::register('support_probe', __FILE__, 'bluesky_support_probe'); + Hook::register('follow', __FILE__, 'bluesky_follow'); + Hook::register('unfollow', __FILE__, 'bluesky_unfollow'); + Hook::register('block', __FILE__, 'bluesky_block'); + Hook::register('unblock', __FILE__, 'bluesky_unblock'); Hook::register('check_item_notification', __FILE__, 'bluesky_check_item_notification'); - // Hook::register('probe_detect', __FILE__, 'bluesky_probe_detect'); + Hook::register('probe_detect', __FILE__, 'bluesky_probe_detect'); Hook::register('item_by_link', __FILE__, 'bluesky_item_by_link'); } @@ -75,15 +96,68 @@ function bluesky_load_config(ConfigFileManager $loader) function bluesky_check_item_notification(array &$notification_data) { - $handle = DI::pConfig()->get($notification_data['uid'], 'bluesky', 'handle'); - $did = DI::pConfig()->get($notification_data['uid'], 'bluesky', 'did'); + $did = DI::pConfig()->get($notification_data['uid'], 'bluesky', 'did'); - if (!empty($handle) && !empty($did)) { - $notification_data['profiles'][] = $handle; + if (!empty($did)) { $notification_data['profiles'][] = $did; } } +function bluesky_probe_detect(array &$hookData) +{ + // Don't overwrite an existing result + if (isset($hookData['result'])) { + return; + } + + // Avoid a lookup for the wrong network + if (!in_array($hookData['network'], ['', Protocol::BLUESKY])) { + return; + } + + $pconfig = DBA::selectFirst('pconfig', ['uid'], ["`cat` = ? AND `k` = ? AND `v` != ?", 'bluesky', 'access_token', '']); + if (empty($pconfig['uid'])) { + return; + } + + if (parse_url($hookData['uri'], PHP_URL_SCHEME) == 'did') { + $did = $hookData['uri']; + } elseif (preg_match('#^' . BLUESKY_WEB . '/profile/(.+)#', $hookData['uri'], $matches)) { + $did = bluesky_get_did($matches[1]); + if (empty($did)) { + return; + } + } else { + return; + } + + $token = bluesky_get_token($pconfig['uid']); + if (empty($token)) { + return; + } + + $data = bluesky_xrpc_get($pconfig['uid'], 'app.bsky.actor.getProfile', ['actor' => $did]); + if (empty($data)) { + return; + } + + $hookData['result'] = bluesky_get_contact_fields($data, 0, false); + + $hookData['result']['baseurl'] = bluesky_get_pds($did); + + // Preparing probe data. This differs slightly from the contact array + $hookData['result']['about'] = HTML::toBBCode($data->description ?? ''); + $hookData['result']['photo'] = $data->avatar ?? ''; + $hookData['result']['header'] = $data->banner ?? ''; + $hookData['result']['batch'] = ''; + $hookData['result']['notify'] = ''; + $hookData['result']['poll'] = ''; + $hookData['result']['poco'] = ''; + $hookData['result']['pubkey'] = ''; + $hookData['result']['priority'] = 0; + $hookData['result']['guid'] = ''; +} + function bluesky_item_by_link(array &$hookData) { // Don't overwrite an existing result @@ -96,11 +170,11 @@ function bluesky_item_by_link(array &$hookData) return; } - if (!preg_match('#^' . BLUESKY_HOST . '/profile/(.+)/post/(.+)#', $hookData['uri'], $matches)) { + if (!preg_match('#^' . BLUESKY_WEB . '/profile/(.+)/post/(.+)#', $hookData['uri'], $matches)) { return; } - $did = bluesky_get_did($hookData['uid'], $matches[1]); + $did = bluesky_get_did($matches[1]); if (empty($did)) { return; } @@ -109,7 +183,7 @@ function bluesky_item_by_link(array &$hookData) $uri = 'at://' . $did . '/app.bsky.feed.post/' . $matches[2]; - $uri = bluesky_fetch_missing_post($uri, $hookData['uid'], 0, true); + $uri = bluesky_fetch_missing_post($uri, $hookData['uid'], $hookData['uid'], 0, 0); Logger::debug('Got post', ['profile' => $matches[1], 'cid' => $matches[2], 'result' => $uri]); if (!empty($uri)) { $item = Post::selectFirst(['id'], ['uri' => $uri, 'uid' => $hookData['uid']]); @@ -119,32 +193,155 @@ function bluesky_item_by_link(array &$hookData) } } +function bluesky_support_follow(array &$data) +{ + if ($data['protocol'] == Protocol::BLUESKY) { + $data['result'] = true; + } +} + +function bluesky_support_probe(array &$data) +{ + if ($data['protocol'] == Protocol::BLUESKY) { + $data['result'] = true; + } +} + +function bluesky_follow(array &$hook_data) +{ + $token = bluesky_get_token($hook_data['uid']); + if (empty($token)) { + return; + } + + Logger::debug('Check if contact is bluesky', ['data' => $hook_data]); + $contact = DBA::selectFirst('contact', [], ['network' => Protocol::BLUESKY, 'url' => $hook_data['url'], 'uid' => [0, $hook_data['uid']]]); + if (empty($contact)) { + return; + } + + $record = [ + 'subject' => $contact['url'], + 'createdAt' => DateTimeFormat::utcNow(DateTimeFormat::ATOM), + '$type' => 'app.bsky.graph.follow' + ]; + + $post = [ + 'collection' => 'app.bsky.graph.follow', + 'repo' => DI::pConfig()->get($hook_data['uid'], 'bluesky', 'did'), + 'record' => $record + ]; + + $activity = bluesky_xrpc_post($hook_data['uid'], 'com.atproto.repo.createRecord', $post); + if (!empty($activity->uri)) { + $hook_data['contact'] = $contact; + Logger::debug('Successfully start following', ['url' => $contact['url'], 'uri' => $activity->uri]); + } +} + +function bluesky_unfollow(array &$hook_data) +{ + $token = bluesky_get_token($hook_data['uid']); + if (empty($token)) { + return; + } + + if ($hook_data['contact']['network'] != Protocol::BLUESKY) { + return; + } + + $data = bluesky_xrpc_get($hook_data['uid'], 'app.bsky.actor.getProfile', ['actor' => $hook_data['contact']['url']]); + if (empty($data->viewer) || empty($data->viewer->following)) { + return; + } + + bluesky_delete_post($data->viewer->following, $hook_data['uid']); + + $hook_data['result'] = true; +} + +function bluesky_block(array &$hook_data) +{ + $token = bluesky_get_token($hook_data['uid']); + if (empty($token)) { + return; + } + + Logger::debug('Check if contact is bluesky', ['data' => $hook_data]); + $contact = DBA::selectFirst('contact', [], ['network' => Protocol::BLUESKY, 'url' => $hook_data['url'], 'uid' => [0, $hook_data['uid']]]); + if (empty($contact)) { + return; + } + + $record = [ + 'subject' => $contact['url'], + 'createdAt' => DateTimeFormat::utcNow(DateTimeFormat::ATOM), + '$type' => 'app.bsky.graph.block' + ]; + + $post = [ + 'collection' => 'app.bsky.graph.block', + 'repo' => DI::pConfig()->get($hook_data['uid'], 'bluesky', 'did'), + 'record' => $record + ]; + + $activity = bluesky_xrpc_post($hook_data['uid'], 'com.atproto.repo.createRecord', $post); + if (!empty($activity->uri)) { + $cdata = Contact::getPublicAndUserContactID($hook_data['contact']['id'], $hook_data['uid']); + if (!empty($cdata['user'])) { + Contact::remove($cdata['user']); + } + Logger::debug('Successfully blocked contact', ['url' => $hook_data['contact']['url'], 'uri' => $activity->uri]); + } +} + +function bluesky_unblock(array &$hook_data) +{ + $token = bluesky_get_token($hook_data['uid']); + if (empty($token)) { + return; + } + + if ($hook_data['contact']['network'] != Protocol::BLUESKY) { + return; + } + + $data = bluesky_xrpc_get($hook_data['uid'], 'app.bsky.actor.getProfile', ['actor' => $hook_data['contact']['url']]); + if (empty($data->viewer) || empty($data->viewer->blocking)) { + return; + } + + bluesky_delete_post($data->viewer->blocking, $hook_data['uid']); + + $hook_data['result'] = true; +} + function bluesky_settings(array &$data) { if (!DI::userSession()->getLocalUserId()) { return; } - $enabled = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'post') ?? false; - $def_enabled = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'post_by_default') ?? false; - $host = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'host') ?: 'https://bsky.social'; - $handle = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'handle'); - $did = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'did'); - $token = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'access_token'); - $import = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'import') ?? false; - - $status = $token ? DI::l10n()->t("You are authenticated to Bluesky. For security reasons the password isn't stored.") : DI::l10n()->t('You are not authenticated. Please enter the app password.'); + $enabled = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'post') ?? false; + $def_enabled = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'post_by_default') ?? false; + $pds = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'pds'); + $handle = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'handle'); + $did = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'did'); + $token = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'access_token'); + $import = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'import') ?? false; + $import_feeds = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'import_feeds') ?? false; $t = Renderer::getMarkupTemplate('connector_settings.tpl', 'addon/bluesky/'); $html = Renderer::replaceMacros($t, [ - '$enable' => ['bluesky', DI::l10n()->t('Enable Bluesky Post Addon'), $enabled], - '$bydefault' => ['bluesky_bydefault', DI::l10n()->t('Post to Bluesky by default'), $def_enabled], - '$import' => ['bluesky_import', DI::l10n()->t('Import the remote timeline'), $import], - '$host' => ['bluesky_host', DI::l10n()->t('Bluesky host'), $host, '', '', 'readonly'], - '$handle' => ['bluesky_handle', DI::l10n()->t('Bluesky handle'), $handle], - '$did' => ['bluesky_did', DI::l10n()->t('Bluesky DID'), $did, DI::l10n()->t('This is the unique identifier. It will be fetched automatically, when the handle is entered.'), '', 'readonly'], - '$password' => ['bluesky_password', DI::l10n()->t('Bluesky app password'), '', DI::l10n()->t("Please don't add your real password here, but instead create a specific app password in the Bluesky settings.")], - '$status' => $status + '$enable' => ['bluesky', DI::l10n()->t('Enable Bluesky Post Addon'), $enabled], + '$bydefault' => ['bluesky_bydefault', DI::l10n()->t('Post to Bluesky by default'), $def_enabled], + '$import' => ['bluesky_import', DI::l10n()->t('Import the remote timeline'), $import], + '$import_feeds' => ['bluesky_import_feeds', DI::l10n()->t('Import the pinned feeds'), $import_feeds, DI::l10n()->t('When activated, Posts will be imported from all the feeds that you pinned in Bluesky.')], + '$pds' => ['bluesky_pds', DI::l10n()->t('Personal Data Server'), $pds, DI::l10n()->t('The personal data server (PDS) is the system that hosts your profile.'), '', 'readonly'], + '$handle' => ['bluesky_handle', DI::l10n()->t('Bluesky handle'), $handle], + '$did' => ['bluesky_did', DI::l10n()->t('Bluesky DID'), $did, DI::l10n()->t('This is the unique identifier. It will be fetched automatically, when the handle is entered.'), '', 'readonly'], + '$password' => ['bluesky_password', DI::l10n()->t('Bluesky app password'), '', DI::l10n()->t("Please don't add your real password here, but instead create a specific app password in the Bluesky settings.")], + '$status' => bluesky_get_status($handle, $did, $pds, $token), ]); $data = [ @@ -156,34 +353,92 @@ function bluesky_settings(array &$data) ]; } +function bluesky_get_status(string $handle = null, string $did = null, string $pds = null, string $token = null): string +{ + if (empty($handle)) { + return DI::l10n()->t('You are not authenticated. Please enter your handle and the app password.'); + } + + $status = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'status') ?? BLUEKSY_STATUS_UNKNOWN; + + // Fallback mechanism for connection that had been established before the introduction of the status + if ($status == BLUEKSY_STATUS_UNKNOWN) { + if (empty($did)) { + $status = BLUEKSY_STATUS_DID_FAIL; + } elseif (empty($pds)) { + $status = BLUEKSY_STATUS_PDS_FAIL; + } elseif (!empty($token)) { + $status = BLUEKSY_STATUS_TOKEN_OK; + } else { + $status = BLUEKSY_STATUS_TOKEN_FAIL; + } + } + + switch ($status) { + case BLUEKSY_STATUS_TOKEN_OK: + return DI::l10n()->t("You are authenticated to Bluesky. For security reasons the password isn't stored."); + case BLUEKSY_STATUS_SUCCESS: + return DI::l10n()->t('The communication with the personal data server service (PDS) is established.'); + case BLUEKSY_STATUS_API_FAIL; + return DI::l10n()->t('Communication issues with the personal data server service (PDS).'); + case BLUEKSY_STATUS_DID_FAIL: + return DI::l10n()->t('The DID for the provided handle could not be detected. Please check if you entered the correct handle.'); + case BLUEKSY_STATUS_PDS_FAIL: + return DI::l10n()->t('The personal data server service (PDS) could not be detected.'); + case BLUEKSY_STATUS_TOKEN_FAIL: + return DI::l10n()->t('The authentication with the provided handle and password failed. Please check if you entered the correct password.'); + default: + return ''; + } +} + function bluesky_settings_post(array &$b) { if (empty($_POST['bluesky-submit'])) { return; } - $old_host = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'host'); + $old_pds = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'pds'); $old_handle = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'handle'); $old_did = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'did'); - $host = $_POST['bluesky_host']; - $handle = $_POST['bluesky_handle']; + $handle = trim($_POST['bluesky_handle'], ' @'); DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'post', intval($_POST['bluesky'])); DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'post_by_default', intval($_POST['bluesky_bydefault'])); - DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'host', $host); DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'handle', $handle); DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'import', intval($_POST['bluesky_import'])); + DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'import_feeds', intval($_POST['bluesky_import_feeds'])); - if (!empty($host) && !empty($handle)) { - if (empty($old_did) || $old_host != $host || $old_handle != $handle) { - DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'did', bluesky_get_did(DI::userSession()->getLocalUserId(), DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'handle'))); + if (!empty($handle)) { + if (empty($old_did) || $old_handle != $handle) { + $did = bluesky_get_did(DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'handle')); + if (empty($did)) { + DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'status', BLUEKSY_STATUS_DID_FAIL); + } + DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'did', $did); + } else { + $did = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'did'); + } + if (!empty($did) && (empty($old_pds) || $old_handle != $handle)) { + $pds = bluesky_get_pds($did); + if (empty($pds)) { + DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'status', BLUEKSY_STATUS_PDS_FAIL); + } + DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'pds', $pds); + } else { + $pds = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'pds'); } } else { DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'bluesky', 'did'); + DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'bluesky', 'pds'); + DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'bluesky', 'access_token'); + DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'bluesky', 'refresh_token'); + DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'bluesky', 'token_created'); + DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'bluesky', 'status'); } - if (!empty($_POST['bluesky_password'])) { + if (!empty($did) && !empty($pds) && !empty($_POST['bluesky_password'])) { bluesky_create_token(DI::userSession()->getLocalUserId(), $_POST['bluesky_password']); } } @@ -208,7 +463,7 @@ function bluesky_jot_nets(array &$jotnets_fields) function bluesky_cron() { - $last = DI::keyValue()->get('bluesky_last_poll'); + $last = (int)DI::keyValue()->get('bluesky_last_poll'); $poll_interval = intval(DI::config()->get('bluesky', 'poll_interval')); if (!$poll_interval) { @@ -240,9 +495,30 @@ function bluesky_cron() } } - Logger::notice('importing timeline - start', ['user' => $pconfig['uid']]); - bluesky_fetch_timeline($pconfig['uid']); - Logger::notice('importing timeline - done', ['user' => $pconfig['uid']]); + // Refresh the token now, so that it doesn't need to be refreshed in parallel by the following workers + bluesky_get_token($pconfig['uid']); + + Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'addon/bluesky/bluesky_timeline.php', $pconfig['uid'], $last); + Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'addon/bluesky/bluesky_notifications.php', $pconfig['uid'], $last); + + if (DI::pConfig()->get($pconfig['uid'], 'bluesky', 'import_feeds')) { + $feeds = bluesky_get_feeds($pconfig['uid']); + foreach ($feeds as $feed) { + Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'addon/bluesky/bluesky_feed.php', $pconfig['uid'], $feed, $last); + } + } + } + + $last_clean = DI::keyValue()->get('bluesky_last_clean'); + if (empty($last_clean) || ($last_clean + 86400 < time())) { + Logger::notice('Start contact cleanup'); + $contacts = DBA::select('account-user-view', ['id', 'pid'], ["`network` = ? AND `uid` != ? AND `rel` = ?", Protocol::BLUESKY, 0, Contact::NOTHING]); + while ($contact = DBA::fetch($contacts)) { + Worker::add(Worker::PRIORITY_LOW, 'MergeContact', $contact['pid'], $contact['id'], 0); + } + DBA::close($contacts); + DI::keyValue()->set('bluesky_last_clean', time()); + Logger::notice('Contact cleanup done'); } Logger::notice('cron_end'); @@ -389,7 +665,7 @@ function bluesky_create_activity(array $item, stdClass $parent = null) ]; } - $activity = bluesky_post($uid, '/xrpc/com.atproto.repo.createRecord', json_encode($post), ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . $token]]); + $activity = bluesky_xrpc_post($uid, 'com.atproto.repo.createRecord', $post); if (empty($activity)) { return; } @@ -407,20 +683,32 @@ function bluesky_create_post(array $item, stdClass $root = null, stdClass $paren return; } + // Try to fetch the language from the post itself + if (!empty($item['language'])) { + $language = array_key_first(json_decode($item['language'], true)); + } else { + $language = ''; + } + $did = DI::pConfig()->get($uid, 'bluesky', 'did'); - $urls = bluesky_get_urls($item['body']); + $urls = bluesky_get_urls(Post\Media::removeFromBody($item['body'])); + $item['body'] = $urls['body']; - $msg = Plaintext::getPost($item, 300, false, BBCode::CONNECTORS); + $msg = Plaintext::getPost($item, 300, false, BBCode::BLUESKY); foreach ($msg['parts'] as $key => $part) { - $facets = bluesky_get_facets($part, $urls); + $facets = bluesky_get_facets($part, $urls['urls']); $record = [ 'text' => $facets['body'], + '$type' => 'app.bsky.feed.post', 'createdAt' => DateTimeFormat::utcNow(DateTimeFormat::ATOM), - '$type' => 'app.bsky.feed.post' ]; + if (!empty($language)) { + $record['langs'] = [$language]; + } + if (!empty($facets['facets'])) { $record['facets'] = $facets['facets']; } @@ -431,6 +719,12 @@ function bluesky_create_post(array $item, stdClass $root = null, stdClass $paren if ($key == count($msg['parts']) - 1) { $record = bluesky_add_embed($uid, $msg, $record); + if (empty($record)) { + if (Worker::getRetrial() < 3) { + Worker::defer(); + } + return; + } } $post = [ @@ -439,8 +733,11 @@ function bluesky_create_post(array $item, stdClass $root = null, stdClass $paren 'record' => $record ]; - $parent = bluesky_post($uid, '/xrpc/com.atproto.repo.createRecord', json_encode($post), ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . $token]]); + $parent = bluesky_xrpc_post($uid, 'com.atproto.repo.createRecord', $post); if (empty($parent)) { + if ($part == 0) { + Worker::defer(); + } return; } Logger::debug('Posting done', ['return' => $parent]); @@ -457,25 +754,61 @@ function bluesky_create_post(array $item, stdClass $root = null, stdClass $paren function bluesky_get_urls(string $body): array { - // Remove all hashtags and mentions - $body = preg_replace("/([#@!])\[url\=(.*?)\](.*?)\[\/url\]/ism", '', $body); + // Remove all hashtag and mention links + $body = preg_replace("/([@!])\[url\=(.*?)\](.*?)\[\/url\]/ism", '$1$3', $body); + $body = BBCode::expandVideoLinks($body); $urls = []; + // Search for hash tags + if (preg_match_all("/#\[url\=(https?:.*?)\](.*?)\[\/url\]/ism", $body, $matches, PREG_SET_ORDER)) { + foreach ($matches as $match) { + $text = '#' . $match[2]; + $urls[] = ['tag' => $match[2], 'text' => $text, 'hash' => $text]; + $body = str_replace($match[0], $text, $body); + } + } + // Search for pure links - if (preg_match_all("/\[url\](https?:.*?)\[\/url\]/ism", $body, $matches)) { - foreach ($matches[1] as $url) { - $urls[] = $url; + if (preg_match_all("/\[url\](https?:.*?)\[\/url\]/ism", $body, $matches, PREG_SET_ORDER)) { + foreach ($matches as $match) { + $text = Strings::getStyledURL($match[1]); + $hash = bluesky_get_hash_for_url($match[0], mb_strlen($text)); + $urls[] = ['url' => $match[1], 'text' => $text, 'hash' => $hash]; + $body = str_replace($match[0], $hash, $body); } } // Search for links with descriptions - if (preg_match_all("/\[url\=(https?:.*?)\].*?\[\/url\]/ism", $body, $matches)) { - foreach ($matches[1] as $url) { - $urls[] = $url; + if (preg_match_all("/\[url\=(https?:.*?)\](.*?)\[\/url\]/ism", $body, $matches, PREG_SET_ORDER)) { + foreach ($matches as $match) { + if ($match[1] == $match[2]) { + $text = Strings::getStyledURL($match[1]); + } else { + $text = $match[2]; + } + if (mb_strlen($text) < 100) { + $hash = bluesky_get_hash_for_url($match[0], mb_strlen($text)); + $urls[] = ['url' => $match[1], 'text' => $text, 'hash' => $hash]; + $body = str_replace($match[0], $hash, $body); + } else { + $text = Strings::getStyledURL($match[1]); + $hash = bluesky_get_hash_for_url($match[0], mb_strlen($text)); + $urls[] = ['url' => $match[1], 'text' => $text, 'hash' => $hash]; + $body = str_replace($match[0], $text . ' ' . $hash, $body); + } } } - return $urls; + + return ['body' => $body, 'urls' => $urls]; +} + +function bluesky_get_hash_for_url(string $text, int $linklength): string +{ + if ($linklength <= 10) { + return '|' . hash('crc32', $text) . '|'; + } + return substr('|' . hash('crc32', $text) . base64_encode($text), 0, $linklength - 2) . '|'; } function bluesky_get_facets(string $body, array $urls): array @@ -483,7 +816,7 @@ function bluesky_get_facets(string $body, array $urls): array $facets = []; foreach ($urls as $url) { - $pos = strpos($body, $url); + $pos = strpos($body, $url['hash']); if ($pos === false) { continue; } @@ -492,18 +825,26 @@ function bluesky_get_facets(string $body, array $urls): array } else { $prefix = ''; } - $linktext = Strings::getStyledURL($url); - $body = $prefix . $linktext . substr($body, $pos + strlen($url)); + + $body = $prefix . $url['text'] . substr($body, $pos + strlen($url['hash'])); $facet = new stdClass; $facet->index = new stdClass; - $facet->index->byteEnd = $pos + strlen($linktext); + $facet->index->byteEnd = $pos + strlen($url['text']); $facet->index->byteStart = $pos; $feature = new stdClass; - $feature->uri = $url; + $type = '$type'; - $feature->$type = 'app.bsky.richtext.facet#link'; + if (!empty($url['tag'])) { + $feature->tag = $url['tag']; + $feature->$type = 'app.bsky.richtext.facet#tag'; + } elseif (!empty($url['url'])) { + $feature->uri = $url['url']; + $feature->$type = 'app.bsky.richtext.facet#link'; + } else { + continue; + } $facet->features = [$feature]; $facets[] = $facet; @@ -517,12 +858,15 @@ function bluesky_add_embed(int $uid, array $msg, array $record): array if (($msg['type'] != 'link') && !empty($msg['images'])) { $images = []; foreach ($msg['images'] as $image) { - $photo = Photo::selectFirst(['resource-id'], ['id' => $image['id']]); - $photo = Photo::selectFirst([], ["`resource-id` = ? AND `scale` > ?", $photo['resource-id'], 0], ['order' => ['scale']]); + if (count($images) == 4) { + continue; + } + $photo = Photo::selectFirst([], ['id' => $image['id']]); $blob = bluesky_upload_blob($uid, $photo); - if (!empty($blob) && count($images) < 4) { - $images[] = ['alt' => $image['description'], 'image' => $blob]; + if (empty($blob)) { + return []; } + $images[] = ['alt' => $image['description'] ?? '', 'image' => $blob]; } if (!empty($images)) { $record['embed'] = ['$type' => 'app.bsky.embed.images', 'images' => $images]; @@ -532,8 +876,8 @@ function bluesky_add_embed(int $uid, array $msg, array $record): array '$type' => 'app.bsky.embed.external', 'external' => [ 'uri' => $msg['url'], - 'title' => $msg['title'], - 'description' => $msg['description'], + 'title' => $msg['title'] ?? '', + 'description' => $msg['description'] ?? '', ] ]; if (!empty($msg['image'])) { @@ -549,31 +893,46 @@ function bluesky_add_embed(int $uid, array $msg, array $record): array function bluesky_upload_blob(int $uid, array $photo): ?stdClass { + $retrial = Worker::getRetrial(); $content = Photo::getImageForPhoto($photo); + + $picture = new Image($content, $photo['type']); + $height = $picture->getHeight(); + $width = $picture->getWidth(); + $size = strlen($content); + + $picture = Photo::resizeToFileSize($picture, BLUESKY_IMAGE_SIZE[$retrial]); + $new_height = $picture->getHeight(); + $new_width = $picture->getWidth(); + $content = $picture->asString(); + $new_size = strlen($content); + + Logger::info('Uploading', ['uid' => $uid, 'retrial' => $retrial, 'height' => $new_height, 'width' => $new_width, 'size' => $new_size, 'orig-height' => $height, 'orig-width' => $width, 'orig-size' => $size]); + $data = bluesky_post($uid, '/xrpc/com.atproto.repo.uploadBlob', $content, ['Content-type' => $photo['type'], 'Authorization' => ['Bearer ' . bluesky_get_token($uid)]]); if (empty($data)) { + Logger::info('Uploading failed', ['uid' => $uid, 'retrial' => $retrial, 'height' => $new_height, 'width' => $new_width, 'size' => $new_size, 'orig-height' => $height, 'orig-width' => $width, 'orig-size' => $size]); return null; } - Logger::debug('Uploaded blob', ['return' => $data]); + Logger::debug('Uploaded blob', ['return' => $data, 'uid' => $uid, 'retrial' => $retrial, 'height' => $new_height, 'width' => $new_width, 'size' => $new_size, 'orig-height' => $height, 'orig-width' => $width, 'orig-size' => $size]); return $data->blob; } function bluesky_delete_post(string $uri, int $uid) { - $token = bluesky_get_token($uid); $parts = bluesky_get_uri_parts($uri); if (empty($parts)) { Logger::debug('No uri delected', ['uri' => $uri]); return; } - bluesky_post($uid, '/xrpc/com.atproto.repo.deleteRecord', json_encode($parts), ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . $token]]); + bluesky_xrpc_post($uid, 'com.atproto.repo.deleteRecord', $parts); Logger::debug('Deleted', ['parts' => $parts]); } -function bluesky_fetch_timeline(int $uid) +function bluesky_fetch_timeline(int $uid, int $last_poll) { - $data = bluesky_get($uid, '/xrpc/app.bsky.feed.getTimeline', HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . bluesky_get_token($uid)]]]); + $data = bluesky_xrpc_get($uid, 'app.bsky.feed.getTimeline'); if (empty($data)) { return; } @@ -583,7 +942,7 @@ function bluesky_fetch_timeline(int $uid) } foreach (array_reverse($data->feed) as $entry) { - bluesky_process_post($entry->post, $uid); + bluesky_process_post($entry->post, $uid, Item::PR_NONE, 0, $last_poll); if (!empty($entry->reason)) { bluesky_process_reason($entry->reason, bluesky_get_uri($entry->post), $uid); } @@ -600,7 +959,7 @@ function bluesky_process_reason(stdClass $reason, string $uri, int $uid) return; } - $contact = bluesky_get_contact($reason->by, $uid); + $contact = bluesky_get_contact($reason->by, $uid, $uid); $item = [ 'network' => Protocol::BLUESKY, @@ -624,6 +983,7 @@ function bluesky_process_reason(stdClass $reason, string $uri, int $uid) return; } + $item['guid'] = Item::guidFromUri($item['uri'], $contact['alias']); $item['owner-name'] = $item['author-name']; $item['owner-link'] = $item['author-link']; $item['owner-avatar'] = $item['author-avatar']; @@ -633,33 +993,157 @@ function bluesky_process_reason(stdClass $reason, string $uri, int $uid) } } -function bluesky_process_post(stdClass $post, int $uid): int +function bluesky_fetch_notifications(int $uid, int $last_poll) +{ + $data = bluesky_xrpc_get($uid, 'app.bsky.notification.listNotifications'); + if (empty($data->notifications)) { + return; + } + foreach ($data->notifications as $notification) { + $uri = bluesky_get_uri($notification); + if (Post::exists(['uri' => $uri, 'uid' => $uid]) || Post::exists(['extid' => $uri, 'uid' => $uid])) { + Logger::debug('Notification already processed', ['uid' => $uid, 'reason' => $notification->reason, 'uri' => $uri, 'indexedAt' => $notification->indexedAt]); + continue; + } + Logger::debug('Process notification', ['uid' => $uid, 'reason' => $notification->reason, 'uri' => $uri, 'indexedAt' => $notification->indexedAt]); + switch ($notification->reason) { + case 'like': + $item = bluesky_get_header($notification, $uri, $uid, $uid); + $item['gravity'] = Item::GRAVITY_ACTIVITY; + $item['body'] = $item['verb'] = Activity::LIKE; + $item['thr-parent'] = bluesky_get_uri($notification->record->subject); + $item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $uid, $item['contact-id'], 0, $last_poll); + if (!empty($item['thr-parent'])) { + $data = Item::insert($item); + Logger::debug('Got like', ['uid' => $uid, 'result' => $data, 'uri' => $uri]); + } else { + Logger::info('Thread parent not found', ['uid' => $uid, 'parent' => $item['thr-parent'], 'uri' => $uri]); + } + break; + + case 'repost': + $item = bluesky_get_header($notification, $uri, $uid, $uid); + $item['gravity'] = Item::GRAVITY_ACTIVITY; + $item['body'] = $item['verb'] = Activity::ANNOUNCE; + $item['thr-parent'] = bluesky_get_uri($notification->record->subject); + $item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $uid, $item['contact-id'], 0, $last_poll); + if (!empty($item['thr-parent'])) { + $data = Item::insert($item); + Logger::debug('Got repost', ['uid' => $uid, 'result' => $data, 'uri' => $uri]); + } else { + Logger::info('Thread parent not found', ['uid' => $uid, 'parent' => $item['thr-parent'], 'uri' => $uri]); + } + break; + + case 'follow': + $contact = bluesky_get_contact($notification->author, $uid, $uid); + Logger::debug('New follower', ['uid' => $uid, 'nick' => $contact['nick'], 'uri' => $uri]); + break; + + case 'mention': + $data = bluesky_process_post($notification, $uid, Item::PR_PUSHED, 0, $last_poll); + Logger::debug('Got mention', ['uid' => $uid, 'result' => $data, 'uri' => $uri]); + break; + + case 'reply': + $data = bluesky_process_post($notification, $uid, Item::PR_PUSHED, 0, $last_poll); + Logger::debug('Got reply', ['uid' => $uid, 'result' => $data, 'uri' => $uri]); + break; + + case 'quote': + $data = bluesky_process_post($notification, $uid, Item::PR_PUSHED, 0, $last_poll); + Logger::debug('Got quote', ['uid' => $uid, 'result' => $data, 'uri' => $uri]); + break; + + default: + Logger::notice('Unhandled reason', ['reason' => $notification->reason, 'uri' => $uri]); + break; + } + } +} + +function bluesky_fetch_feed(int $uid, string $feed, int $last_poll) +{ + $data = bluesky_xrpc_get($uid, 'app.bsky.feed.getFeed', ['feed' => $feed]); + if (empty($data)) { + return; + } + + if (empty($data->feed)) { + return; + } + + $feeddata = bluesky_xrpc_get($uid, 'app.bsky.feed.getFeedGenerator', ['feed' => $feed]); + if (!empty($feeddata)) { + $feedurl = $feeddata->view->uri; + $feedname = $feeddata->view->displayName; + } else { + $feedurl = $feed; + $feedname = $feed; + } + + foreach (array_reverse($data->feed) as $entry) { + $contact = bluesky_get_contact($entry->post->author, 0, $uid); + $languages = $entry->post->record->langs ?? []; + + if (!Relay::isWantedLanguage($entry->post->record->text, 0, $contact['id'] ?? 0, $languages)) { + Logger::debug('Unwanted language detected', ['text' => $entry->post->record->text]); + continue; + } + $id = bluesky_process_post($entry->post, $uid, Item::PR_TAG, 0, $last_poll); + if (!empty($id)) { + $post = Post::selectFirst(['uri-id'], ['id' => $id]); + if (!empty($post['uri-id'])) { + $stored = Post\Category::storeFileByURIId($post['uri-id'], $uid, Post\Category::SUBCRIPTION, $feedname, $feedurl); + Logger::debug('Stored tag subscription for user', ['uri-id' => $post['uri-id'], 'uid' => $uid, 'name' => $feedname, 'url' => $feedurl, 'stored' => $stored]); + } else { + Logger::notice('Post not found', ['id' => $id, 'entry' => $entry]); + } + } + if (!empty($entry->reason)) { + bluesky_process_reason($entry->reason, bluesky_get_uri($entry->post), $uid); + } + } +} + +function bluesky_process_post(stdClass $post, int $uid, int $post_reason, int $level, int $last_poll): int { $uri = bluesky_get_uri($post); - if (Post::exists(['uri' => $uri, 'uid' => $uid]) || Post::exists(['extid' => $uri, 'uid' => $uid])) { - return 0; + if ($id = Post::selectFirst(['id'], ['uri' => $uri, 'uid' => $uid])) { + return $id['id']; } - Logger::debug('Importing post', ['uid' => $uid, 'indexedAt' => $post->indexedAt, 'uri' => $post->uri, 'cid' => $post->cid]); + if ($id = Post::selectFirst(['id'], ['extid' => $uri, 'uid' => $uid])) { + return $id['id']; + } - $item = bluesky_get_header($post, $uri, $uid); + Logger::debug('Importing post', ['uid' => $uid, 'indexedAt' => $post->indexedAt, 'uri' => $post->uri, 'cid' => $post->cid, 'root' => $post->record->reply->root ?? '']); - $item = bluesky_get_content($item, $post->record, $uid); + $item = bluesky_get_header($post, $uri, $uid, $uid); + $item = bluesky_get_content($item, $post->record, $uri, $uid, $uid, $level, $last_poll); + if (empty($item)) { + return 0; + } if (!empty($post->embed)) { - $item = bluesky_add_media($post->embed, $item); + $item = bluesky_add_media($post->embed, $item, $uid, $level, $last_poll); } - return item::insert($item); + + if (empty($item['post-reason'])) { + $item['post-reason'] = $post_reason; + } + + return Item::insert($item); } -function bluesky_get_header(stdClass $post, string $uri, int $uid): array +function bluesky_get_header(stdClass $post, string $uri, int $uid, int $fetch_uid): array { $parts = bluesky_get_uri_parts($uri); if (empty($post->author)) { return []; } - $contact = bluesky_get_contact($post->author, $uid); + $contact = bluesky_get_contact($post->author, $uid, $fetch_uid); $item = [ 'network' => Protocol::BLUESKY, 'uid' => $uid, @@ -673,6 +1157,7 @@ function bluesky_get_header(stdClass $post, string $uri, int $uid): array 'author-link' => $contact['url'], 'author-avatar' => $contact['avatar'], 'plink' => $contact['alias'] . '/post/' . $parts->rkey, + 'source' => json_encode($post), ]; $item['uri-id'] = ItemURI::getIdByURI($uri); @@ -680,26 +1165,51 @@ function bluesky_get_header(stdClass $post, string $uri, int $uid): array $item['owner-link'] = $item['author-link']; $item['owner-avatar'] = $item['author-avatar']; + if (in_array($contact['rel'], [Contact::SHARING, Contact::FRIEND])) { + $item['post-reason'] = Item::PR_FOLLOWER; + } + return $item; } -function bluesky_get_content(array $item, stdClass $record, int $uid): array +function bluesky_get_content(array $item, stdClass $record, string $uri, int $uid, int $fetch_uid, int $level, int $last_poll): array { + if (empty($item)) { + return []; + } + if (!empty($record->reply)) { $item['parent-uri'] = bluesky_get_uri($record->reply->root); - $item['parent-uri'] = bluesky_fetch_missing_post($item['parent-uri'], $uid, $item['contact-id']); + if ($item['parent-uri'] != $uri) { + $item['parent-uri'] = bluesky_fetch_missing_post($item['parent-uri'], $uid, $fetch_uid, $item['contact-id'], $level, $last_poll); + if (empty($item['parent-uri'])) { + return []; + } + } + $item['thr-parent'] = bluesky_get_uri($record->reply->parent); - $item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $item['contact-id']); + if (!in_array($item['thr-parent'], [$uri, $item['parent-uri']])) { + $item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $fetch_uid, $item['contact-id'], $level, $last_poll, $item['parent-uri']); + if (empty($item['thr-parent'])) { + return []; + } + } } - $item['body'] = bluesky_get_text($record, $uid); + $item['body'] = bluesky_get_text($record, $item['uri-id']); $item['created'] = DateTimeFormat::utc($record->createdAt, DateTimeFormat::MYSQL); + $item['transmitted-languages'] = $record->langs ?? []; + + if (($last_poll != 0) && strtotime($item['created']) > $last_poll) { + $item['received'] = $item['created']; + } + return $item; } -function bluesky_get_text(stdClass $record, int $uid): string +function bluesky_get_text(stdClass $record, int $uri_id): string { - $text = $record->text; + $text = $record->text ?? ''; if (empty($record->facets)) { return $text; @@ -716,21 +1226,35 @@ function bluesky_get_text(stdClass $record, int $uid): string $linktext = substr($text, $facet->index->byteStart, $facet->index->byteEnd - $facet->index->byteStart); $suffix = substr($text, $facet->index->byteEnd); - $url = ''; - + $url = ''; + $type = '$type'; foreach ($facet->features as $feature) { - if (!empty($feature->uri)) { - $url = $feature->uri; - } - if (!empty($feature->did)) { - $contact = Contact::selectFirst(['id'], ['nurl' => $feature->did, 'uid' => [0, $uid]]); - if (!empty($contact['id'])) { - $url = DI::baseUrl() . '/contact/' . $contact['id']; - if (substr($linktext, 0, 1) == '@') { - $prefix .= '@'; - $linktext = substr($linktext, 1); - } - } + + switch ($feature->$type) { + case 'app.bsky.richtext.facet#link': + $url = $feature->uri; + break; + + case 'app.bsky.richtext.facet#mention': + $contact = Contact::getByURL($feature->did, null, ['id']); + if (!empty($contact['id'])) { + $url = DI::baseUrl() . '/contact/' . $contact['id']; + if (substr($linktext, 0, 1) == '@') { + $prefix .= '@'; + $linktext = substr($linktext, 1); + } + } + break; + + case 'app.bsky.richtext.facet#tag'; + Tag::store($uri_id, Tag::HASHTAG, $feature->tag); + $url = DI::baseUrl() . '/search?tag=' . urlencode($feature->tag); + $linktext = '#' . $feature->tag; + break; + + default: + Logger::notice('Unhandled feature type', ['type' => $feature->$type, 'feature' => $feature, 'record' => $record]); + break; } } if (!empty($url)) { @@ -740,56 +1264,95 @@ function bluesky_get_text(stdClass $record, int $uid): string return $text; } -function bluesky_add_media(stdClass $embed, array $item): array +function bluesky_add_media(stdClass $embed, array $item, int $fetch_uid, int $level, int $last_poll): array { - if (!empty($embed->images)) { - foreach ($embed->images as $image) { + $type = '$type'; + switch ($embed->$type) { + case 'app.bsky.embed.images#view': + foreach ($embed->images as $image) { + $media = [ + 'uri-id' => $item['uri-id'], + 'type' => Post\Media::IMAGE, + 'url' => $image->fullsize, + 'preview' => $image->thumb, + 'description' => $image->alt, + ]; + Post\Media::insert($media); + } + break; + + case 'app.bsky.embed.external#view': $media = [ - 'uri-id' => $item['uri-id'], - 'type' => Post\Media::IMAGE, - 'url' => $image->fullsize, - 'preview' => $image->thumb, - 'description' => $image->alt, + 'uri-id' => $item['uri-id'], + 'type' => Post\Media::HTML, + 'url' => $embed->external->uri, + 'name' => $embed->external->title, + 'description' => $embed->external->description, ]; Post\Media::insert($media); - } - } elseif (!empty($embed->external)) { - $media = [ - 'uri-id' => $item['uri-id'], - 'type' => Post\Media::HTML, - 'url' => $embed->external->uri, - 'name' => $embed->external->title, - 'description' => $embed->external->description, - ]; - Post\Media::insert($media); - } elseif (!empty($embed->record)) { - $uri = bluesky_get_uri($embed->record); - $shared = Post::selectFirst(['uri-id'], ['uri' => $uri, 'uid' => $item['uid']]); - if (empty($shared)) { - $shared = bluesky_get_header($embed->record, $uri, 0); - if (!empty($shared)) { - $shared = bluesky_get_content($shared, $embed->record->value, $item['uid']); - - if (!empty($embed->record->embeds)) { - foreach ($embed->record->embeds as $single) { - $shared = bluesky_add_media($single, $shared); + break; + + case 'app.bsky.embed.record#view': + $uri = bluesky_get_uri($embed->record); + $shared = Post::selectFirst(['uri-id'], ['uri' => $uri, 'uid' => $item['uid']]); + if (empty($shared)) { + if (empty($embed->record->value)) { + Logger::info('Record has got no value', ['record' => $embed->record]); + break; + } + $shared = bluesky_get_header($embed->record, $uri, 0, $fetch_uid); + $shared = bluesky_get_content($shared, $embed->record->value, $uri, $item['uid'], $fetch_uid, $level, $last_poll); + if (!empty($shared)) { + if (!empty($embed->record->embeds)) { + foreach ($embed->record->embeds as $single) { + $shared = bluesky_add_media($single, $shared, $fetch_uid, $level, $last_poll); + } } + Item::insert($shared); } - $id = Item::insert($shared); - $shared = Post::selectFirst(['uri-id'], ['id' => $id]); } - } - if (!empty($shared)) { - $item['quote-uri-id'] = $shared['uri-id']; - } - } else { - Logger::debug('Unsupported embed', ['embed' => $embed, 'item' => $item]); + if (!empty($shared['uri-id'])) { + $item['quote-uri-id'] = $shared['uri-id']; + } + break; + + case 'app.bsky.embed.recordWithMedia#view': + $uri = bluesky_get_uri($embed->record->record); + $shared = Post::selectFirst(['uri-id'], ['uri' => $uri, 'uid' => $item['uid']]); + if (empty($shared)) { + $shared = bluesky_get_header($embed->record->record, $uri, 0, $fetch_uid); + $shared = bluesky_get_content($shared, $embed->record->record->value, $uri, $item['uid'], $fetch_uid, $level, $last_poll); + if (!empty($shared)) { + if (!empty($embed->record->record->embeds)) { + foreach ($embed->record->record->embeds as $single) { + $shared = bluesky_add_media($single, $shared, $fetch_uid, $level, $last_poll); + } + } + Item::insert($shared); + } + } + if (!empty($shared['uri-id'])) { + $item['quote-uri-id'] = $shared['uri-id']; + } + + if (!empty($embed->media)) { + $item = bluesky_add_media($embed->media, $item, $fetch_uid, $level, $last_poll); + } + break; + + default: + Logger::notice('Unhandled embed type', ['type' => $embed->$type, 'embed' => $embed]); + break; } return $item; } function bluesky_get_uri(stdClass $post): string { + if (empty($post->cid)) { + Logger::info('Invalid URI', ['post' => $post]); + return ''; + } return $post->uri . ':' . $post->cid; } @@ -810,6 +1373,11 @@ function bluesky_get_uri_class(string $uri): ?stdClass $class->cid = array_pop($elements); $class->uri = implode(':', $elements); + if ((substr_count($class->uri, '/') == 2) && (substr_count($class->cid, '/') == 2)) { + $class->uri .= ':' . $class->cid; + $class->cid = ''; + } + return $class; } @@ -831,7 +1399,42 @@ function bluesky_get_uri_parts(string $uri): ?stdClass return $class; } -function bluesky_fetch_missing_post(string $uri, int $uid, int $causer, bool $original = false): string +function bluesky_fetch_missing_post(string $uri, int $uid, int $fetch_uid, int $causer, int $level, int $last_poll = 0, string $fallback = ''): string +{ + $fetched_uri = bluesky_fetch_post($uri, $uid); + if (!empty($fetched_uri)) { + return $fetched_uri; + } + + if (++$level > 100) { + Logger::info('Recursion level too deep', ['level' => $level, 'uid' => $uid, 'uri' => $uri, 'fallback' => $fallback]); + // When the level is too deep we will fallback to the parent uri. + // Allthough the threading won't be correct, we at least had stored all posts and won't try again + return $fallback; + } + + $class = bluesky_get_uri_class($uri); + $fetch_uri = $class->uri; + + Logger::debug('Fetch missing post', ['level' => $level, 'uid' => $uid, 'uri' => $uri]); + $data = bluesky_xrpc_get($fetch_uid, 'app.bsky.feed.getPostThread', ['uri' => $fetch_uri]); + if (empty($data)) { + Logger::info('Thread was not fetched', ['level' => $level, 'uid' => $uid, 'uri' => $uri, 'fallback' => $fallback]); + return $fallback; + } + + Logger::debug('Reply count', ['level' => $level, 'uid' => $uid, 'uri' => $uri]); + + if ($causer != 0) { + $cdata = Contact::getPublicAndUserContactID($causer, $uid); + } else { + $cdata = []; + } + + return bluesky_process_thread($data->thread, $uid, $fetch_uid, $cdata, $level, $last_poll); +} + +function bluesky_fetch_post(string $uri, int $uid): string { if (Post::exists(['uri' => $uri, 'uid' => [$uid, 0]])) { Logger::debug('Post exists', ['uri' => $uri]); @@ -840,93 +1443,107 @@ function bluesky_fetch_missing_post(string $uri, int $uid, int $causer, bool $or $reply = Post::selectFirst(['uri'], ['extid' => $uri, 'uid' => [$uid, 0]]); if (!empty($reply['uri'])) { + Logger::debug('Post with extid exists', ['uri' => $uri]); return $reply['uri']; } + return ''; +} - Logger::debug('Fetch missing post', ['uri' => $uri]); - if (!$original) { - $class = bluesky_get_uri_class($uri); - $fetch_uri = $class->uri; - } else { - $fetch_uri = $uri; - } - - $data = bluesky_get($uid, '/xrpc/app.bsky.feed.getPosts?uris=' . urlencode($fetch_uri), HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . bluesky_get_token($uid)]]]); - if (empty($data)) { +function bluesky_process_thread(stdClass $thread, int $uid, int $fetch_uid, array $cdata, int $level, int $last_poll): string +{ + if (empty($thread->post)) { + Logger::info('Invalid post', ['post' => $thread]); return ''; } + $uri = bluesky_get_uri($thread->post); - if ($causer != 0) { - $cdata = Contact::getPublicAndUserContactID($causer, $uid); - } - - foreach ($data->posts as $post) { - $uri = bluesky_get_uri($post); - $item = bluesky_get_header($post, $uri, $uid); - $item = bluesky_get_content($item, $post->record, $uid); + $fetched_uri = bluesky_fetch_post($uri, $uid); + if (empty($fetched_uri)) { + Logger::debug('Process missing post', ['uri' => $uri]); + $item = bluesky_get_header($thread->post, $uri, $uid, $uid); + $item = bluesky_get_content($item, $thread->post->record, $uri, $uid, $fetch_uid, $level, $last_poll); + if (!empty($item)) { + $item['post-reason'] = Item::PR_FETCHED; - $item['post-reason'] = Item::PR_FETCHED; + if (!empty($cdata['public'])) { + $item['causer-id'] = $cdata['public']; + } - if (!empty($cdata['public'])) { - $item['causer-id'] = $cdata['public']; + if (!empty($thread->post->embed)) { + $item = bluesky_add_media($thread->post->embed, $item, $uid, $level, $last_poll); + } + $id = Item::insert($item); + if (!$id) { + Logger::info('Item has not not been stored', ['uri' => $uri]); + return ''; + } + Logger::debug('Stored item', ['id' => $id, 'uri' => $uri]); + } else { + Logger::info('Post has not not been fetched', ['uri' => $uri]); + return ''; } + } else { + Logger::debug('Post exists', ['uri' => $uri]); + $uri = $fetched_uri; + } - if (!empty($post->embed)) { - $item = bluesky_add_media($post->embed, $item); - } - $id = Item::insert($item); - Logger::debug('Stored item', ['id' => $id, 'uri' => $uri]); + foreach ($thread->replies ?? [] as $reply) { + $reply_uri = bluesky_process_thread($reply, $uid, $fetch_uid, $cdata, $level, $last_poll); + Logger::debug('Reply has been processed', ['uri' => $uri, 'reply' => $reply_uri]); } return $uri; } -function bluesky_get_contact(stdClass $author, int $uid): array +function bluesky_get_contact(stdClass $author, int $uid, int $fetch_uid): array { - $condition = ['network' => Protocol::BLUESKY, 'uid' => $uid, 'url' => $author->did]; + $condition = ['network' => Protocol::BLUESKY, 'uid' => 0, 'url' => $author->did]; + $contact = Contact::selectFirst(['id', 'updated'], $condition); - $fields = [ - 'alias' => BLUESKY_HOST . '/profile/' . $author->handle, - 'name' => $author->displayName, - 'nick' => $author->handle, - 'addr' => $author->handle, - ]; + $update = empty($contact) || $contact['updated'] < DateTimeFormat::utc('now -24 hours'); + + $public_fields = $fields = bluesky_get_contact_fields($author, $fetch_uid, $update); - $contact = Contact::selectFirst([], $condition); + $public_fields['uid'] = 0; + $public_fields['rel'] = Contact::NOTHING; if (empty($contact)) { - $cid = bluesky_insert_contact($author, $uid); + $cid = Contact::insert($public_fields); } else { $cid = $contact['id']; - if ($fields['alias'] != $contact['alias'] || $fields['name'] != $contact['name'] || $fields['nick'] != $contact['nick'] || $fields['addr'] != $contact['addr']) { - Contact::update($fields, ['id' => $cid]); - } + Contact::update($public_fields, ['id' => $cid], true); } - $condition['uid'] = 0; + if ($uid != 0) { + $condition = ['network' => Protocol::BLUESKY, 'uid' => $uid, 'url' => $author->did]; - $contact = Contact::selectFirst([], $condition); - if (empty($contact)) { - $pcid = bluesky_insert_contact($author, 0); - } else { - $pcid = $contact['id']; - if ($fields['alias'] != $contact['alias'] || $fields['name'] != $contact['name'] || $fields['nick'] != $contact['nick'] || $fields['addr'] != $contact['addr']) { - Contact::update($fields, ['id' => $pcid]); + $contact = Contact::selectFirst(['id', 'rel', 'uid'], $condition); + if (!isset($fields['rel']) && isset($contact['rel'])) { + $fields['rel'] = $contact['rel']; + } elseif (!isset($fields['rel'])) { + $fields['rel'] = Contact::NOTHING; } } + if (($uid != 0) && ($fields['rel'] != Contact::NOTHING)) { + if (empty($contact)) { + $cid = Contact::insert($fields); + } else { + $cid = $contact['id']; + Contact::update($fields, ['id' => $cid], true); + } + Logger::debug('Get user contact', ['id' => $cid, 'uid' => $uid, 'update' => $update]); + } else { + Logger::debug('Get public contact', ['id' => $cid, 'uid' => $uid, 'update' => $update]); + } if (!empty($author->avatar)) { Contact::updateAvatar($cid, $author->avatar); } - if (empty($contact) || $contact['updated'] < DateTimeFormat::utc('now -24 hours')) { - bluesky_update_contact($author, $uid, $cid, $pcid); - } - return Contact::getById($cid); } -function bluesky_insert_contact(stdClass $author, int $uid) +function bluesky_get_contact_fields(stdClass $author, int $uid, bool $update): array { $fields = [ 'uid' => $uid, @@ -938,28 +1555,30 @@ function bluesky_insert_contact(stdClass $author, int $uid) 'pending' => false, 'url' => $author->did, 'nurl' => $author->did, - 'alias' => BLUESKY_HOST . '/profile/' . $author->handle, - 'name' => $author->displayName, + 'alias' => BLUESKY_WEB . '/profile/' . $author->handle, + 'name' => $author->displayName ?? $author->handle, 'nick' => $author->handle, 'addr' => $author->handle, ]; - return Contact::insert($fields); -} -function bluesky_update_contact(stdClass $author, int $uid, int $cid, int $pcid) -{ - $data = bluesky_get($uid, '/xrpc/app.bsky.actor.getProfile?actor=' . $author->did, HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . bluesky_get_token($uid)]]]); + if (!$update) { + Logger::debug('Got contact fields', ['uid' => $uid, 'url' => $fields['url']]); + return $fields; + } + + $fields['baseurl'] = bluesky_get_pds($author->did); + if (!empty($fields['baseurl'])) { + GServer::check($fields['baseurl'], Protocol::BLUESKY); + $fields['gsid'] = GServer::getID($fields['baseurl'], true); + } + + $data = bluesky_xrpc_get($uid, 'app.bsky.actor.getProfile', ['actor' => $author->did]); if (empty($data)) { - return; + Logger::debug('Error fetching contact fields', ['uid' => $uid, 'url' => $fields['url']]); + return $fields; } - $fields = [ - 'alias' => BLUESKY_HOST . '/profile/' . $data->handle, - 'name' => $data->displayName, - 'nick' => $data->handle, - 'addr' => $data->handle, - 'updated' => DateTimeFormat::utcNow(DateTimeFormat::MYSQL), - ]; + $fields['updated'] = DateTimeFormat::utcNow(DateTimeFormat::MYSQL); if (!empty($data->description)) { $fields['about'] = HTML::toBBCode($data->description); @@ -969,13 +1588,51 @@ function bluesky_update_contact(stdClass $author, int $uid, int $cid, int $pcid) $fields['header'] = $data->banner; } - Contact::update($fields, ['id' => $cid]); - Contact::update($fields, ['id' => $pcid]); + if (!empty($data->viewer)) { + if (!empty($data->viewer->following) && !empty($data->viewer->followedBy)) { + $fields['rel'] = Contact::FRIEND; + } elseif (!empty($data->viewer->following) && empty($data->viewer->followedBy)) { + $fields['rel'] = Contact::SHARING; + } elseif (empty($data->viewer->following) && !empty($data->viewer->followedBy)) { + $fields['rel'] = Contact::FOLLOWER; + } else { + $fields['rel'] = Contact::NOTHING; + } + } + + Logger::debug('Got updated contact fields', ['uid' => $uid, 'url' => $fields['url']]); + return $fields; } -function bluesky_get_did(int $uid, string $handle): string +function bluesky_get_feeds(int $uid): array { - $data = bluesky_get($uid, '/xrpc/com.atproto.identity.resolveHandle?handle=' . $handle); + $type = '$type'; + $preferences = bluesky_get_preferences($uid); + foreach ($preferences->preferences as $preference) { + if ($preference->$type == 'app.bsky.actor.defs#savedFeedsPref') { + return $preference->pinned ?? []; + } + } + return []; +} + +function bluesky_get_preferences(int $uid): stdClass +{ + $cachekey = 'bluesky:preferences:' . $uid; + $data = DI::cache()->get($cachekey); + if (!is_null($data)) { + return $data; + } + + $data = bluesky_xrpc_get($uid, 'app.bsky.actor.getPreferences'); + + DI::cache()->set($cachekey, $data, Duration::HOUR); + return $data; +} + +function bluesky_get_did(string $handle): string +{ + $data = bluesky_get(BLUESKY_PDS . '/xrpc/com.atproto.identity.resolveHandle?handle=' . urlencode($handle)); if (empty($data)) { return ''; } @@ -983,6 +1640,38 @@ function bluesky_get_did(int $uid, string $handle): string return $data->did; } +function bluesky_get_user_pds(int $uid): string +{ + $pds = DI::pConfig()->get($uid, 'bluesky', 'pds'); + if (!empty($pds)) { + return $pds; + } + $did = DI::pConfig()->get($uid, 'bluesky', 'did'); + if (empty($did)) { + Logger::notice('Empty did for user', ['uid' => $uid]); + return ''; + } + $pds = bluesky_get_pds($did); + DI::pConfig()->set($uid, 'bluesky', 'pds', $pds); + return $pds; +} + +function bluesky_get_pds(string $did): ?string +{ + $data = bluesky_get(BLUESKY_DIRECTORY . '/' . $did); + if (empty($data) || empty($data->service)) { + return null; + } + + foreach ($data->service as $service) { + if (($service->id == '#atproto_pds') && ($service->type == 'AtprotoPersonalDataServer') && !empty($service->serviceEndpoint)) { + return $service->serviceEndpoint; + } + } + + return null; +} + function bluesky_get_token(int $uid): string { $token = DI::pConfig()->get($uid, 'bluesky', 'access_token'); @@ -1019,6 +1708,7 @@ function bluesky_create_token(int $uid, string $password): string $data = bluesky_post($uid, '/xrpc/com.atproto.server.createSession', json_encode(['identifier' => $did, 'password' => $password]), ['Content-type' => 'application/json']); if (empty($data)) { + DI::pConfig()->set($uid, 'bluesky', 'status', BLUEKSY_STATUS_TOKEN_FAIL); return ''; } @@ -1026,30 +1716,51 @@ function bluesky_create_token(int $uid, string $password): string DI::pConfig()->set($uid, 'bluesky', 'access_token', $data->accessJwt); DI::pConfig()->set($uid, 'bluesky', 'refresh_token', $data->refreshJwt); DI::pConfig()->set($uid, 'bluesky', 'token_created', time()); + DI::pConfig()->set($uid, 'bluesky', 'status', BLUEKSY_STATUS_TOKEN_OK); return $data->accessJwt; } +function bluesky_xrpc_post(int $uid, string $url, $parameters): ?stdClass +{ + return bluesky_post($uid, '/xrpc/' . $url, json_encode($parameters), ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . bluesky_get_token($uid)]]); +} + function bluesky_post(int $uid, string $url, string $params, array $headers): ?stdClass { try { - $curlResult = DI::httpClient()->post(DI::pConfig()->get($uid, 'bluesky', 'host') . $url, $params, $headers); + $curlResult = DI::httpClient()->post(bluesky_get_user_pds($uid) . $url, $params, $headers); } catch (\Exception $e) { Logger::notice('Exception on post', ['exception' => $e]); + DI::pConfig()->set($uid, 'bluesky', 'status', BLUEKSY_STATUS_API_FAIL); return null; } if (!$curlResult->isSuccess()) { Logger::notice('API Error', ['error' => json_decode($curlResult->getBody()) ?: $curlResult->getBody()]); + DI::pConfig()->set($uid, 'bluesky', 'status', BLUEKSY_STATUS_API_FAIL); return null; } + DI::pConfig()->set($uid, 'bluesky', 'status', BLUEKSY_STATUS_SUCCESS); return json_decode($curlResult->getBody()); } -function bluesky_get(int $uid, string $url, string $accept_content = HttpClientAccept::DEFAULT, array $opts = []): ?stdClass +function bluesky_xrpc_get(int $uid, string $url, array $parameters = []): ?stdClass +{ + if (!empty($parameters)) { + $url .= '?' . http_build_query($parameters); + } + + $data = bluesky_get(bluesky_get_user_pds($uid) . '/xrpc/' . $url, HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . bluesky_get_token($uid)]]]); + DI::pConfig()->set($uid, 'bluesky', 'status', is_null($data) ? BLUEKSY_STATUS_API_FAIL : BLUEKSY_STATUS_SUCCESS); + return $data; + +} + +function bluesky_get(string $url, string $accept_content = HttpClientAccept::DEFAULT, array $opts = []): ?stdClass { try { - $curlResult = DI::httpClient()->get(DI::pConfig()->get($uid, 'bluesky', 'host') . $url, $accept_content, $opts); + $curlResult = DI::httpClient()->get($url, $accept_content, $opts); } catch (\Exception $e) { Logger::notice('Exception on get', ['exception' => $e]); return null;