X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=plugins%2FTwitterBridge%2Fdaemons%2Ftwitterstatusfetcher.php;h=9298d9e3a1cd5543d05d5c49e20cb5e1bd8f7c00;hb=62dfdb34a613f61e8f546b60468e1a73ee18d7e0;hp=cf75e48f68b0a873fa33a90feb5f58ec4d5261ad;hpb=4d70ba9597e211a7650630d3627ca8b427e2f96c;p=quix0rs-gnu-social.git diff --git a/plugins/TwitterBridge/daemons/twitterstatusfetcher.php b/plugins/TwitterBridge/daemons/twitterstatusfetcher.php index cf75e48f68..9298d9e3a1 100755 --- a/plugins/TwitterBridge/daemons/twitterstatusfetcher.php +++ b/plugins/TwitterBridge/daemons/twitterstatusfetcher.php @@ -40,7 +40,6 @@ require_once INSTALLDIR . '/scripts/commandline.inc'; require_once INSTALLDIR . '/lib/common.php'; require_once INSTALLDIR . '/lib/daemon.php'; require_once INSTALLDIR . '/plugins/TwitterBridge/twitter.php'; -require_once INSTALLDIR . '/plugins/TwitterBridge/twitterbasicauthclient.php'; require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php'; /** @@ -63,7 +62,6 @@ require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php'; * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 * @link http://status.net/ */ - class TwitterStatusFetcher extends ParallelizingDaemon { /** @@ -88,7 +86,6 @@ class TwitterStatusFetcher extends ParallelizingDaemon * * @return string Name of the daemon. */ - function name() { return ('twitterstatusfetcher.'.$this->_id); @@ -100,11 +97,9 @@ class TwitterStatusFetcher extends ParallelizingDaemon * * @return array flinks an array of Foreign_link objects */ - function getObjects() { global $_DB_DATAOBJECT; - $flink = new Foreign_link(); $conn = &$flink->getDatabaseConnection(); @@ -135,12 +130,10 @@ class TwitterStatusFetcher extends ParallelizingDaemon } function childTask($flink) { - // Each child ps needs its own DB connection // Note: DataObject::getDatabaseConnection() creates // a new connection if there isn't one already - $conn = &$flink->getDatabaseConnection(); $this->getTimeline($flink); @@ -152,7 +145,6 @@ class TwitterStatusFetcher extends ParallelizingDaemon // XXX: Couldn't find a less brutal way to blow // away a cached connection - global $_DB_DATAOBJECT; unset($_DB_DATAOBJECT['CONNECTIONS']); } @@ -168,10 +160,6 @@ class TwitterStatusFetcher extends ParallelizingDaemon common_debug($this->name() . ' - Trying to get timeline for Twitter user ' . $flink->foreign_id); - // XXX: Biggest remaining issue - How do we know at which status - // to start importing? How many statuses? Right now I'm going - // with the default last 20. - $client = null; if (TwitterOAuthClient::isPackedToken($flink->credentials)) { @@ -179,14 +167,17 @@ class TwitterStatusFetcher extends ParallelizingDaemon $client = new TwitterOAuthClient($token->key, $token->secret); common_debug($this->name() . ' - Grabbing friends timeline with OAuth.'); } else { - $client = new TwitterBasicAuthClient($flink); - common_debug($this->name() . ' - Grabbing friends timeline with basic auth.'); + common_debug("Skipping friends timeline for $flink->foreign_id since not OAuth."); } $timeline = null; + $lastId = Twitter_synch_status::getLastId($flink->foreign_id, 'home_timeline'); + + common_debug("Got lastId value '{$lastId}' for foreign id '{$flink->foreign_id}' and timeline 'home_timeline'"); + try { - $timeline = $client->statusesFriendsTimeline(); + $timeline = $client->statusesHomeTimeline($lastId); } catch (Exception $e) { common_log(LOG_WARNING, $this->name() . ' - Twitter client unable to get friends timeline for user ' . @@ -201,461 +192,27 @@ class TwitterStatusFetcher extends ParallelizingDaemon common_debug(LOG_INFO, $this->name() . ' - Retrieved ' . sizeof($timeline) . ' statuses from Twitter.'); + $importer = new TwitterImport(); + // Reverse to preserve order foreach (array_reverse($timeline) as $status) { + $notice = $importer->importStatus($status); - // Hacktastic: filter out stuff coming from this StatusNet - - $source = mb_strtolower(common_config('integration', 'source')); - - if (preg_match("/$source/", mb_strtolower($status->source))) { - common_debug($this->name() . ' - Skipping import of status ' . - $status->id . ' with source ' . $source); - continue; - } - - // Don't save it if the user is protected - // FIXME: save it but treat it as private - - if ($status->user->protected) { - continue; + if (!empty($notice)) { + Inbox::insertNotice($flink->user_id, $notice->id); } + } - $this->saveStatus($status, $flink); + if (!empty($timeline)) { + Twitter_synch_status::setLastId($flink->foreign_id, 'home_timeline', $timeline[0]->id); + common_debug("Set lastId value '{$timeline[0]->id}' for foreign id '{$flink->foreign_id}' and timeline 'home_timeline'"); } // Okay, record the time we synced with Twitter for posterity - $flink->last_noticesync = common_sql_now(); $flink->update(); } - - function saveStatus($status, $flink) - { - $profile = $this->ensureProfile($status->user); - - if (empty($profile)) { - common_log(LOG_ERR, $this->name() . - ' - Problem saving notice. No associated Profile.'); - return; - } - - $statusUri = $this->makeStatusURI($status->user->screen_name, $status->id); - - // check to see if we've already imported the status - - $dupe = $this->checkDupe($profile, $statusUri); - - if (!empty($dupe)) { - common_log( - LOG_INFO, - $this->name() . - " - Ignoring duplicate import: $statusUri" - ); - return; - } - - $notice = new Notice(); - - $notice->profile_id = $profile->id; - $notice->uri = $statusUri; - $notice->url = $statusUri; - $notice->created = strftime( - '%Y-%m-%d %H:%M:%S', - strtotime($status->created_at) - ); - - $notice->source = 'twitter'; - - $notice->reply_to = null; - - if (!empty($status->in_reply_to_status_id)) { - $replyUri = $this->makeStatusURI($status->in_reply_to_screen_name, $status->in_reply_to_status_id); - $reply = Notice::staticGet('uri', $replyUri); - if (!empty($reply)) { - $notice->reply_to = $reply->id; - $notice->conversation = $reply->conversation; - } - } - - if (empty($notice->conversation)) { - $conv = Conversation::create(); - $notice->conversation = $conv->id; - } - - $notice->is_local = Notice::GATEWAY; - - $notice->content = common_shorten_links($status->text); - $notice->rendered = common_render_content( - $notice->content, - $notice - ); - - if (Event::handle('StartNoticeSave', array(&$notice))) { - - $id = $notice->insert(); - - if (!$id) { - common_log_db_error($notice, 'INSERT', __FILE__); - common_log(LOG_ERR, $this->name() . - ' - Problem saving notice.'); - } - - Event::handle('EndNoticeSave', array($notice)); - } - - Inbox::insertNotice($flink->user_id, $notice->id); - $notice->blowOnInsert(); - - return $notice; - } - - /** - * Make an URI for a status. - * - * @param object $status status object - * - * @return string URI - */ - - function makeStatusURI($username, $id) - { - return 'http://twitter.com/' - . $username - . '/status/' - . $id; - } - - /** - * Look up a Profile by profileurl field. Profile::staticGet() was - * not working consistently. - * - * @param string $nickname local nickname of the Twitter user - * @param string $profileurl the profile url - * - * @return mixed value the first Profile with that url, or null - */ - - function getProfileByUrl($nickname, $profileurl) - { - $profile = new Profile(); - $profile->nickname = $nickname; - $profile->profileurl = $profileurl; - $profile->limit(1); - - if ($profile->find()) { - $profile->fetch(); - return $profile; - } - - return null; - } - - /** - * Check to see if this Twitter status has already been imported - * - * @param Profile $profile Twitter user's local profile - * @param string $statusUri URI of the status on Twitter - * - * @return mixed value a matching Notice or null - */ - - function checkDupe($profile, $statusUri) - { - $notice = new Notice(); - $notice->uri = $statusUri; - $notice->profile_id = $profile->id; - $notice->limit(1); - - if ($notice->find()) { - $notice->fetch(); - return $notice; - } - - return null; - } - - function ensureProfile($user) - { - // check to see if there's already a profile for this user - - $profileurl = 'http://twitter.com/' . $user->screen_name; - $profile = $this->getProfileByUrl($user->screen_name, $profileurl); - - if (!empty($profile)) { - common_debug($this->name() . - " - Profile for $profile->nickname found."); - - // Check to see if the user's Avatar has changed - - $this->checkAvatar($user, $profile); - return $profile; - - } else { - - common_debug($this->name() . ' - Adding profile and remote profile ' . - "for Twitter user: $profileurl."); - - $profile = new Profile(); - $profile->query("BEGIN"); - - $profile->nickname = $user->screen_name; - $profile->fullname = $user->name; - $profile->homepage = $user->url; - $profile->bio = $user->description; - $profile->location = $user->location; - $profile->profileurl = $profileurl; - $profile->created = common_sql_now(); - - try { - $id = $profile->insert(); - } catch(Exception $e) { - common_log(LOG_WARNING, $this->name . ' Couldn\'t insert profile - ' . $e->getMessage()); - } - - if (empty($id)) { - common_log_db_error($profile, 'INSERT', __FILE__); - $profile->query("ROLLBACK"); - return false; - } - - // check for remote profile - - $remote_pro = Remote_profile::staticGet('uri', $profileurl); - - if (empty($remote_pro)) { - - $remote_pro = new Remote_profile(); - - $remote_pro->id = $id; - $remote_pro->uri = $profileurl; - $remote_pro->created = common_sql_now(); - - try { - $rid = $remote_pro->insert(); - } catch (Exception $e) { - common_log(LOG_WARNING, $this->name() . ' Couldn\'t save remote profile - ' . $e->getMessage()); - } - - if (empty($rid)) { - common_log_db_error($profile, 'INSERT', __FILE__); - $profile->query("ROLLBACK"); - return false; - } - } - - $profile->query("COMMIT"); - - $this->saveAvatars($user, $id); - - return $profile; - } - } - - function checkAvatar($twitter_user, $profile) - { - global $config; - - $path_parts = pathinfo($twitter_user->profile_image_url); - - $newname = 'Twitter_' . $twitter_user->id . '_' . - $path_parts['basename']; - - $oldname = $profile->getAvatar(48)->filename; - - if ($newname != $oldname) { - common_debug($this->name() . ' - Avatar for Twitter user ' . - "$profile->nickname has changed."); - common_debug($this->name() . " - old: $oldname new: $newname"); - - $this->updateAvatars($twitter_user, $profile); - } - - if ($this->missingAvatarFile($profile)) { - common_debug($this->name() . ' - Twitter user ' . - $profile->nickname . - ' is missing one or more local avatars.'); - common_debug($this->name() ." - old: $oldname new: $newname"); - - $this->updateAvatars($twitter_user, $profile); - } - } - - function updateAvatars($twitter_user, $profile) { - - global $config; - - $path_parts = pathinfo($twitter_user->profile_image_url); - - $img_root = substr($path_parts['basename'], 0, -11); - $ext = $path_parts['extension']; - $mediatype = $this->getMediatype($ext); - - foreach (array('mini', 'normal', 'bigger') as $size) { - $url = $path_parts['dirname'] . '/' . - $img_root . '_' . $size . ".$ext"; - $filename = 'Twitter_' . $twitter_user->id . '_' . - $img_root . "_$size.$ext"; - - $this->updateAvatar($profile->id, $size, $mediatype, $filename); - $this->fetchAvatar($url, $filename); - } - } - - function missingAvatarFile($profile) { - foreach (array(24, 48, 73) as $size) { - $filename = $profile->getAvatar($size)->filename; - $avatarpath = Avatar::path($filename); - if (file_exists($avatarpath) == FALSE) { - return true; - } - } - return false; - } - - function getMediatype($ext) - { - $mediatype = null; - - switch (strtolower($ext)) { - case 'jpg': - $mediatype = 'image/jpg'; - break; - case 'gif': - $mediatype = 'image/gif'; - break; - default: - $mediatype = 'image/png'; - } - - return $mediatype; - } - - function saveAvatars($user, $id) - { - global $config; - - $path_parts = pathinfo($user->profile_image_url); - $ext = $path_parts['extension']; - $end = strlen('_normal' . $ext); - $img_root = substr($path_parts['basename'], 0, -($end+1)); - $mediatype = $this->getMediatype($ext); - - foreach (array('mini', 'normal', 'bigger') as $size) { - $url = $path_parts['dirname'] . '/' . - $img_root . '_' . $size . ".$ext"; - $filename = 'Twitter_' . $user->id . '_' . - $img_root . "_$size.$ext"; - - if ($this->fetchAvatar($url, $filename)) { - $this->newAvatar($id, $size, $mediatype, $filename); - } else { - common_log(LOG_WARNING, $id() . - " - Problem fetching Avatar: $url"); - } - } - } - - function updateAvatar($profile_id, $size, $mediatype, $filename) { - - common_debug($this->name() . " - Updating avatar: $size"); - - $profile = Profile::staticGet($profile_id); - - if (empty($profile)) { - common_debug($this->name() . " - Couldn't get profile: $profile_id!"); - return; - } - - $sizes = array('mini' => 24, 'normal' => 48, 'bigger' => 73); - $avatar = $profile->getAvatar($sizes[$size]); - - // Delete the avatar, if present - - if ($avatar) { - $avatar->delete(); - } - - $this->newAvatar($profile->id, $size, $mediatype, $filename); - } - - function newAvatar($profile_id, $size, $mediatype, $filename) - { - global $config; - - $avatar = new Avatar(); - $avatar->profile_id = $profile_id; - - switch($size) { - case 'mini': - $avatar->width = 24; - $avatar->height = 24; - break; - case 'normal': - $avatar->width = 48; - $avatar->height = 48; - break; - default: - - // Note: Twitter's big avatars are a different size than - // StatusNet's (StatusNet's = 96) - - $avatar->width = 73; - $avatar->height = 73; - } - - $avatar->original = 0; // we don't have the original - $avatar->mediatype = $mediatype; - $avatar->filename = $filename; - $avatar->url = Avatar::url($filename); - - $avatar->created = common_sql_now(); - - try { - $id = $avatar->insert(); - } catch (Exception $e) { - common_log(LOG_WARNING, $this->name() . ' Couldn\'t insert avatar - ' . $e->getMessage()); - } - - if (empty($id)) { - common_log_db_error($avatar, 'INSERT', __FILE__); - return null; - } - - common_debug($this->name() . - " - Saved new $size avatar for $profile_id."); - - return $id; - } - - /** - * Fetch a remote avatar image and save to local storage. - * - * @param string $url avatar source URL - * @param string $filename bare local filename for download - * @return bool true on success, false on failure - */ - function fetchAvatar($url, $filename) - { - common_debug($this->name() . " - Fetching Twitter avatar: $url"); - - $request = HTTPClient::start(); - $response = $request->get($url); - if ($response->isOk()) { - $avatarfile = Avatar::path($filename); - $ok = file_put_contents($avatarfile, $response->getBody()); - if (!$ok) { - common_log(LOG_WARNING, $this->name() . - " - Couldn't open file $filename"); - return false; - } - } else { - return false; - } - - return true; - } } $id = null; @@ -677,4 +234,3 @@ if (have_option('d') || have_option('debug')) { $fetcher = new TwitterStatusFetcher($id, 60, 2, $debug); $fetcher->runOnce(); -