Bluesky/Tumblr: Set "received" to "created" if fetched after previous poll

diff --git a/bluesky/bluesky.php b/bluesky/bluesky.php
index 2f70ed745c2d481f5e993ae51e7309f3eb1d07fa..94d5cb3193246e2717ae419b8a636dd885ee132c 100644 (file)
--- a/bluesky/bluesky.php
+++ b/bluesky/bluesky.php
@@ -498,13 +498,13 @@ function bluesky_cron()
                // Refresh the token now, so that it doesn't need to be refreshed in parallel by the following workers
                bluesky_get_token($pconfig['uid']);
 
-               Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'addon/bluesky/bluesky_timeline.php', $pconfig['uid']);
-               Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'addon/bluesky/bluesky_notifications.php', $pconfig['uid']);
+               Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'addon/bluesky/bluesky_timeline.php', $pconfig['uid'], $last);
+               Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'addon/bluesky/bluesky_notifications.php', $pconfig['uid'], $last);
 
                if (DI::pConfig()->get($pconfig['uid'], 'bluesky', 'import_feeds')) {
                        $feeds = bluesky_get_feeds($pconfig['uid']);
                        foreach ($feeds as $feed) {
-                               Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'addon/bluesky/bluesky_feed.php', $pconfig['uid'], $feed);
+                               Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'addon/bluesky/bluesky_feed.php', $pconfig['uid'], $feed, $last);
                        }
                }
        }
@@ -930,7 +930,7 @@ function bluesky_delete_post(string $uri, int $uid)
        Logger::debug('Deleted', ['parts' => $parts]);
 }
 
-function bluesky_fetch_timeline(int $uid)
+function bluesky_fetch_timeline(int $uid, int $last_poll)
 {
        $data = bluesky_xrpc_get($uid, 'app.bsky.feed.getTimeline');
        if (empty($data)) {
@@ -942,7 +942,7 @@ function bluesky_fetch_timeline(int $uid)
        }
 
        foreach (array_reverse($data->feed) as $entry) {
-               bluesky_process_post($entry->post, $uid, Item::PR_NONE, 0);
+               bluesky_process_post($entry->post, $uid, Item::PR_NONE, 0, $last_poll);
                if (!empty($entry->reason)) {
                        bluesky_process_reason($entry->reason, bluesky_get_uri($entry->post), $uid);
                }
@@ -993,7 +993,7 @@ function bluesky_process_reason(stdClass $reason, string $uri, int $uid)
        }
 }
 
-function bluesky_fetch_notifications(int $uid)
+function bluesky_fetch_notifications(int $uid, int $last_poll)
 {
        $data = bluesky_xrpc_get($uid, 'app.bsky.notification.listNotifications');
        if (empty($data->notifications)) {
@@ -1012,7 +1012,7 @@ function bluesky_fetch_notifications(int $uid)
                                $item['gravity'] = Item::GRAVITY_ACTIVITY;
                                $item['body'] = $item['verb'] = Activity::LIKE;
                                $item['thr-parent'] = bluesky_get_uri($notification->record->subject);
-                               $item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $uid, $item['contact-id'], 0);
+                               $item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $uid, $item['contact-id'], 0, $last_poll);
                                if (!empty($item['thr-parent'])) {
                                        $data = Item::insert($item);
                                        Logger::debug('Got like', ['uid' => $uid, 'result' => $data, 'uri' => $uri]);
@@ -1026,7 +1026,7 @@ function bluesky_fetch_notifications(int $uid)
                                $item['gravity'] = Item::GRAVITY_ACTIVITY;
                                $item['body'] = $item['verb'] = Activity::ANNOUNCE;
                                $item['thr-parent'] = bluesky_get_uri($notification->record->subject);
-                               $item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $uid, $item['contact-id'], 0);
+                               $item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $uid, $item['contact-id'], 0, $last_poll);
                                if (!empty($item['thr-parent'])) {
                                        $data = Item::insert($item);
                                        Logger::debug('Got repost', ['uid' => $uid, 'result' => $data, 'uri' => $uri]);
@@ -1041,17 +1041,17 @@ function bluesky_fetch_notifications(int $uid)
                                break;
 
                        case 'mention':
-                               $data = bluesky_process_post($notification, $uid, Item::PR_PUSHED, 0);
+                               $data = bluesky_process_post($notification, $uid, Item::PR_PUSHED, 0, $last_poll);
                                Logger::debug('Got mention', ['uid' => $uid, 'result' => $data, 'uri' => $uri]);
                                break;
 
                        case 'reply':
-                               $data = bluesky_process_post($notification, $uid, Item::PR_PUSHED, 0);
+                               $data = bluesky_process_post($notification, $uid, Item::PR_PUSHED, 0, $last_poll);
                                Logger::debug('Got reply', ['uid' => $uid, 'result' => $data, 'uri' => $uri]);
                                break;
 
                        case 'quote':
-                               $data = bluesky_process_post($notification, $uid, Item::PR_PUSHED, 0);
+                               $data = bluesky_process_post($notification, $uid, Item::PR_PUSHED, 0, $last_poll);
                                Logger::debug('Got quote', ['uid' => $uid, 'result' => $data, 'uri' => $uri]);
                                break;
 
@@ -1062,7 +1062,7 @@ function bluesky_fetch_notifications(int $uid)
        }
 }
 
-function bluesky_fetch_feed(int $uid, string $feed)
+function bluesky_fetch_feed(int $uid, string $feed, int $last_poll)
 {
        $data = bluesky_xrpc_get($uid, 'app.bsky.feed.getFeed', ['feed' => $feed]);
        if (empty($data)) {
@@ -1090,7 +1090,7 @@ function bluesky_fetch_feed(int $uid, string $feed)
                        Logger::debug('Unwanted language detected', ['text' => $entry->post->record->text]);
                        continue;
                }
-               $id = bluesky_process_post($entry->post, $uid, Item::PR_TAG, 0);
+               $id = bluesky_process_post($entry->post, $uid, Item::PR_TAG, 0, $last_poll);
                if (!empty($id)) {
                        $post = Post::selectFirst(['uri-id'], ['id' => $id]);
                        if (!empty($post['uri-id'])) {
@@ -1106,7 +1106,7 @@ function bluesky_fetch_feed(int $uid, string $feed)
        }
 }
 
-function bluesky_process_post(stdClass $post, int $uid, int $post_reason, $level): int
+function bluesky_process_post(stdClass $post, int $uid, int $post_reason, int $level, int $last_poll): int
 {
        $uri = bluesky_get_uri($post);
 
@@ -1121,20 +1121,20 @@ function bluesky_process_post(stdClass $post, int $uid, int $post_reason, $level
        Logger::debug('Importing post', ['uid' => $uid, 'indexedAt' => $post->indexedAt, 'uri' => $post->uri, 'cid' => $post->cid, 'root' => $post->record->reply->root ?? '']);
 
        $item = bluesky_get_header($post, $uri, $uid, $uid);
-       $item = bluesky_get_content($item, $post->record, $uri, $uid, $uid, $level);
+       $item = bluesky_get_content($item, $post->record, $uri, $uid, $uid, $level, $last_poll);
        if (empty($item)) {
                return 0;
        }
 
        if (!empty($post->embed)) {
-               $item = bluesky_add_media($post->embed, $item, $uid, $level);
+               $item = bluesky_add_media($post->embed, $item, $uid, $level, $last_poll);
        }
 
        if (empty($item['post-reason'])) {
                $item['post-reason'] = $post_reason;
        }
 
-       return item::insert($item);
+       return Item::insert($item);
 }
 
 function bluesky_get_header(stdClass $post, string $uri, int $uid, int $fetch_uid): array
@@ -1172,7 +1172,7 @@ function bluesky_get_header(stdClass $post, string $uri, int $uid, int $fetch_ui
        return $item;
 }
 
-function bluesky_get_content(array $item, stdClass $record, string $uri, int $uid, int $fetch_uid, int $level): array
+function bluesky_get_content(array $item, stdClass $record, string $uri, int $uid, int $fetch_uid, int $level, int $last_poll): array
 {
        if (empty($item)) {
                return [];
@@ -1181,7 +1181,7 @@ function bluesky_get_content(array $item, stdClass $record, string $uri, int $ui
        if (!empty($record->reply)) {
                $item['parent-uri'] = bluesky_get_uri($record->reply->root);
                if ($item['parent-uri'] != $uri) {
-                       $item['parent-uri'] = bluesky_fetch_missing_post($item['parent-uri'], $uid, $fetch_uid, $item['contact-id'], $level);
+                       $item['parent-uri'] = bluesky_fetch_missing_post($item['parent-uri'], $uid, $fetch_uid, $item['contact-id'], $level, $last_poll);
                        if (empty($item['parent-uri'])) {
                                return [];
                        }
@@ -1189,7 +1189,7 @@ function bluesky_get_content(array $item, stdClass $record, string $uri, int $ui
 
                $item['thr-parent'] = bluesky_get_uri($record->reply->parent);
                if (!in_array($item['thr-parent'], [$uri, $item['parent-uri']])) {
-                       $item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $fetch_uid, $item['contact-id'], $level, $item['parent-uri']);
+                       $item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $fetch_uid, $item['contact-id'], $level, $last_poll, $item['parent-uri']);
                        if (empty($item['thr-parent'])) {
                                return [];
                        }
@@ -1199,6 +1199,11 @@ function bluesky_get_content(array $item, stdClass $record, string $uri, int $ui
        $item['body']    = bluesky_get_text($record, $item['uri-id']);
        $item['created'] = DateTimeFormat::utc($record->createdAt, DateTimeFormat::MYSQL);
        $item['transmitted-languages'] = $record->langs ?? [];
+
+       if (($last_poll != 0) && strtotime($item['created']) > $last_poll) {
+               $item['received'] = $item['created'];
+       }
+
        return $item;
 }
 
@@ -1259,7 +1264,7 @@ function bluesky_get_text(stdClass $record, int $uri_id): string
        return $text;
 }
 
-function bluesky_add_media(stdClass $embed, array $item, int $fetch_uid, int $level): array
+function bluesky_add_media(stdClass $embed, array $item, int $fetch_uid, int $level, int $last_poll): array
 {
        $type = '$type';
        switch ($embed->$type) {
@@ -1296,11 +1301,11 @@ function bluesky_add_media(stdClass $embed, array $item, int $fetch_uid, int $le
                                        break;
                                }
                                $shared = bluesky_get_header($embed->record, $uri, 0, $fetch_uid);
-                               $shared = bluesky_get_content($shared, $embed->record->value, $uri, $item['uid'], $fetch_uid, $level);
+                               $shared = bluesky_get_content($shared, $embed->record->value, $uri, $item['uid'], $fetch_uid, $level, $last_poll);
                                if (!empty($shared)) {
                                        if (!empty($embed->record->embeds)) {
                                                foreach ($embed->record->embeds as $single) {
-                                                       $shared = bluesky_add_media($single, $shared, $fetch_uid, $level);
+                                                       $shared = bluesky_add_media($single, $shared, $fetch_uid, $level, $last_poll);
                                                }
                                        }
                                        Item::insert($shared);
@@ -1316,11 +1321,11 @@ function bluesky_add_media(stdClass $embed, array $item, int $fetch_uid, int $le
                        $shared = Post::selectFirst(['uri-id'], ['uri' => $uri, 'uid' => $item['uid']]);
                        if (empty($shared)) {
                                $shared = bluesky_get_header($embed->record->record, $uri, 0, $fetch_uid);
-                               $shared = bluesky_get_content($shared, $embed->record->record->value, $uri, $item['uid'], $fetch_uid, $level);
+                               $shared = bluesky_get_content($shared, $embed->record->record->value, $uri, $item['uid'], $fetch_uid, $level, $last_poll);
                                if (!empty($shared)) {
                                        if (!empty($embed->record->record->embeds)) {
                                                foreach ($embed->record->record->embeds as $single) {
-                                                       $shared = bluesky_add_media($single, $shared, $fetch_uid, $level);
+                                                       $shared = bluesky_add_media($single, $shared, $fetch_uid, $level, $last_poll);
                                                }
                                        }
                                        Item::insert($shared);
@@ -1331,7 +1336,7 @@ function bluesky_add_media(stdClass $embed, array $item, int $fetch_uid, int $le
                        }
 
                        if (!empty($embed->media)) {
-                               $item = bluesky_add_media($embed->media, $item, $fetch_uid, $level);
+                               $item = bluesky_add_media($embed->media, $item, $fetch_uid, $level, $last_poll);
                        }
                        break;
 
@@ -1394,7 +1399,7 @@ function bluesky_get_uri_parts(string $uri): ?stdClass
        return $class;
 }
 
-function bluesky_fetch_missing_post(string $uri, int $uid, int $fetch_uid, int $causer, int $level, string $fallback = ''): string
+function bluesky_fetch_missing_post(string $uri, int $uid, int $fetch_uid, int $causer, int $level, int $last_poll = 0, string $fallback = ''): string
 {
        $fetched_uri = bluesky_fetch_post($uri, $uid);
        if (!empty($fetched_uri)) {
@@ -1426,7 +1431,7 @@ function bluesky_fetch_missing_post(string $uri, int $uid, int $fetch_uid, int $
                $cdata = [];
        }
 
-       return bluesky_process_thread($data->thread, $uid, $fetch_uid, $cdata, $level);
+       return bluesky_process_thread($data->thread, $uid, $fetch_uid, $cdata, $level, $last_poll);
 }
 
 function bluesky_fetch_post(string $uri, int $uid): string
@@ -1444,7 +1449,7 @@ function bluesky_fetch_post(string $uri, int $uid): string
        return '';
 }
 
-function bluesky_process_thread(stdClass $thread, int $uid, int $fetch_uid, array $cdata, int $level): string
+function bluesky_process_thread(stdClass $thread, int $uid, int $fetch_uid, array $cdata, int $level, int $last_poll): string
 {
        if (empty($thread->post)) {
                Logger::info('Invalid post', ['post' => $thread]);
@@ -1456,7 +1461,7 @@ function bluesky_process_thread(stdClass $thread, int $uid, int $fetch_uid, arra
        if (empty($fetched_uri)) {
                Logger::debug('Process missing post', ['uri' => $uri]);
                $item = bluesky_get_header($thread->post, $uri, $uid, $uid);
-               $item = bluesky_get_content($item, $thread->post->record, $uri, $uid, $fetch_uid, $level);
+               $item = bluesky_get_content($item, $thread->post->record, $uri, $uid, $fetch_uid, $level, $last_poll);
                if (!empty($item)) {
                        $item['post-reason'] = Item::PR_FETCHED;
 
@@ -1465,7 +1470,7 @@ function bluesky_process_thread(stdClass $thread, int $uid, int $fetch_uid, arra
                        }
 
                        if (!empty($thread->post->embed)) {
-                               $item = bluesky_add_media($thread->post->embed, $item, $uid, $level);
+                               $item = bluesky_add_media($thread->post->embed, $item, $uid, $level, $last_poll);
                        }
                        $id = Item::insert($item);
                        if (!$id) {
@@ -1483,7 +1488,7 @@ function bluesky_process_thread(stdClass $thread, int $uid, int $fetch_uid, arra
        }
 
        foreach ($thread->replies ?? [] as $reply) {
-               $reply_uri = bluesky_process_thread($reply, $uid, $fetch_uid, $cdata, $level);
+               $reply_uri = bluesky_process_thread($reply, $uid, $fetch_uid, $cdata, $level, $last_poll);
                Logger::debug('Reply has been processed', ['uri' => $uri, 'reply' => $reply_uri]);
        }
 

diff --git a/bluesky/bluesky_feed.php b/bluesky/bluesky_feed.php
index 905fadfa71cfe73a4778a5899d903275b9ac3345..c9a54223f629aa1137b55d68087e775c1c860d90 100644 (file)
--- a/bluesky/bluesky_feed.php
+++ b/bluesky/bluesky_feed.php
@@ -6,11 +6,11 @@ function bluesky_feed_run($argv, $argc)
 {
        require_once 'addon/bluesky/bluesky.php';
 
-       if ($argc != 3) {
+       if ($argc != 4) {
                return;
        }
 
-       Logger::debug('Importing feed - start', ['user' => $argv[1], 'feed' => $argv[2]]);
-       bluesky_fetch_feed($argv[1], $argv[2]);
-       Logger::debug('Importing feed - done', ['user' => $argv[1], 'feed' => $argv[2]]);
+       Logger::debug('Importing feed - start', ['user' => $argv[1], 'feed' => $argv[2], 'last_poll' => $argv[3]]);
+       bluesky_fetch_feed($argv[1], $argv[2], $argv[3]);
+       Logger::debug('Importing feed - done', ['user' => $argv[1], 'feed' => $argv[2], 'last_poll' => $argv[3]]);
 }

diff --git a/bluesky/bluesky_notifications.php b/bluesky/bluesky_notifications.php
index 1377abbfe8dbe1327e6594dfc1e0f6558456c741..ad35dfe0fb651979e54ec749e9a8830e50cb059f 100644 (file)
--- a/bluesky/bluesky_notifications.php
+++ b/bluesky/bluesky_notifications.php
@@ -6,11 +6,11 @@ function bluesky_notifications_run($argv, $argc)
 {
        require_once 'addon/bluesky/bluesky.php';
 
-       if ($argc != 2) {
+       if ($argc != 3) {
                return;
        }
 
-       Logger::notice('importing notifications - start', ['user' => $argv[1]]);
-       bluesky_fetch_notifications($argv[1]);
-       Logger::notice('importing notifications - done', ['user' => $argv[1]]);
+       Logger::notice('importing notifications - start', ['user' => $argv[1], 'last_poll' => $argv[2]]);
+       bluesky_fetch_notifications($argv[1], $argv[2]);
+       Logger::notice('importing notifications - done', ['user' => $argv[1], 'last_poll' => $argv[2]]);
 }

diff --git a/bluesky/bluesky_timeline.php b/bluesky/bluesky_timeline.php
index c5dfba26519cbd4fa63d2331dcc38f044f3bdbc3..22ef2224e92ffcac3b0dee1624144d6c6c325ce5 100644 (file)
--- a/bluesky/bluesky_timeline.php
+++ b/bluesky/bluesky_timeline.php
@@ -6,11 +6,11 @@ function bluesky_timeline_run($argv, $argc)
 {
        require_once 'addon/bluesky/bluesky.php';
 
-       if ($argc != 2) {
+       if ($argc != 3) {
                return;
        }
 
-       Logger::notice('importing timeline - start', ['user' => $argv[1]]);
-       bluesky_fetch_timeline($argv[1]);
-       Logger::notice('importing timeline - done', ['user' => $argv[1]]);
+       Logger::notice('importing timeline - start', ['user' => $argv[1], 'last_poll' => $argv[2]]);
+       bluesky_fetch_timeline($argv[1], $argv[2]);
+       Logger::notice('importing timeline - done', ['user' => $argv[1], 'last_poll' => $argv[2]]);
 }

diff --git a/tumblr/tumblr.php b/tumblr/tumblr.php
index 812b5b30c6aa639905e29b8f55be42e0feaec537..9b29deb31454df3969e7aeaf9985bf845dfb2597 100644 (file)
--- a/tumblr/tumblr.php
+++ b/tumblr/tumblr.php
@@ -439,8 +439,8 @@ function tumblr_cron()
                }
 
                Logger::notice('importing timeline - start', ['user' => $pconfig['uid']]);
-               tumblr_fetch_dashboard($pconfig['uid']);
-               tumblr_fetch_tags($pconfig['uid']);
+               tumblr_fetch_dashboard($pconfig['uid'], $last);
+               tumblr_fetch_tags($pconfig['uid'], $last);
                Logger::notice('importing timeline - done', ['user' => $pconfig['uid']]);
        }
 
@@ -719,10 +719,11 @@ function tumblr_get_post_from_uri(string $uri): array
 /**
  * Fetch posts for user defined hashtags for the given user
  *
- * @param integer $uid
+ * @param int $uid
+ * @param int $last_poll
  * @return void
  */
-function tumblr_fetch_tags(int $uid)
+function tumblr_fetch_tags(int $uid, int $last_poll)
 {
        if (!DI::config()->get('tumblr', 'max_tags') ?? TUMBLR_DEFAULT_MAXIMUM_TAGS) {
                return;
@@ -731,7 +732,7 @@ function tumblr_fetch_tags(int $uid)
        foreach (DI::pConfig()->get($uid, 'tumblr', 'tags') ?? [] as $tag) {
                $data = tumblr_get($uid, 'tagged', ['tag' => $tag]);
                foreach (array_reverse($data->response) as $post) {
-                       $id = tumblr_process_post($post, $uid, Item::PR_TAG);
+                       $id = tumblr_process_post($post, $uid, Item::PR_TAG, $last_poll);
                        if (!empty($id)) {
                                Logger::debug('Tag post imported', ['tag' => $tag, 'id' => $id]);
                                $post = Post::selectFirst(['uri-id'], ['id' => $id]);
@@ -745,10 +746,11 @@ function tumblr_fetch_tags(int $uid)
 /**
  * Fetch the dashboard (timeline) for the given user
  *
- * @param integer $uid
+ * @param int $uid
+ * @param int $last_poll
  * @return void
  */
-function tumblr_fetch_dashboard(int $uid)
+function tumblr_fetch_dashboard(int $uid, int $last_poll)
 {
        $parameters = ['reblog_info' => false, 'notes_info' => false, 'npf' => false];
 
@@ -774,13 +776,13 @@ function tumblr_fetch_dashboard(int $uid)
 
                Logger::debug('Importing post', ['uid' => $uid, 'created' => date(DateTimeFormat::MYSQL, $post->timestamp), 'id' => $post->id_string]);
 
-               tumblr_process_post($post, $uid, Item::PR_NONE);
+               tumblr_process_post($post, $uid, Item::PR_NONE, $last_poll);
 
                DI::pConfig()->set($uid, 'tumblr', 'last_id', $last);
        }
 }
 
-function tumblr_process_post(stdClass $post, int $uid, int $post_reason): int
+function tumblr_process_post(stdClass $post, int $uid, int $post_reason, int $last_poll = 0): int
 {
        $uri = 'tumblr::' . $post->id_string . ':' . $post->reblog_key;
 
@@ -798,7 +800,11 @@ function tumblr_process_post(stdClass $post, int $uid, int $post_reason): int
                $item['post-reason'] = Item::PR_FOLLOWER;
        }
 
-       $id = item::insert($item);
+       if (($last_poll != 0) && strtotime($item['created']) > $last_poll) {
+               $item['received'] = $item['created'];
+       }
+
+       $id = Item::insert($item);
 
        if ($id) {
                $stored = Post::selectFirst(['uri-id'], ['id' => $id]);