X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=scripts%2Ftwitterstatusfetcher.php;h=e1745cfc089614f41aa8a5766f179e72a924f836;hb=749d9bfbbf65d962804bea1fa510039da185179f;hp=a61ce1b0d1622f2d63065d0ce118ca77d97ce3a6;hpb=ebc16062ed2ba6a68d96fbe564c54a3b92b4ef68;p=quix0rs-gnu-social.git diff --git a/scripts/twitterstatusfetcher.php b/scripts/twitterstatusfetcher.php index a61ce1b0d1..e1745cfc08 100755 --- a/scripts/twitterstatusfetcher.php +++ b/scripts/twitterstatusfetcher.php @@ -1,8 +1,8 @@ #!/usr/bin/env php . */ -// Abort if called from a web server -if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { - print "This script must be run from the command line\n"; - exit(); -} - define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); -define('LACONICA', true); // Tune number of processes and how often to poll Twitter // XXX: Should these things be in config.php? define('MAXCHILDREN', 2); define('POLL_INTERVAL', 60); // in seconds -// Uncomment this to get useful logging -define('SCRIPT_DEBUG', true); +$shortoptions = 'di::'; +$longoptions = array('id::', 'debug'); + +$helptext = << + * @author Evan Prodromou + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://laconi.ca/ + */ + +// NOTE: an Avatar path MUST be set in config.php for this +// script to work: e.g.: $config['avatar']['path'] = '/laconica/avatar'; class TwitterStatusFetcher extends Daemon { + private $_children = array(); - private $children = array(); + function __construct($id=null, $daemonize=true) + { + parent::__construct($daemonize); + + if ($id) { + $this->set_id($id); + } + } + + /** + * Name of this daemon + * + * @return string Name of the daemon. + */ function name() { - return ('twitterstatusfetcher.generic'); + return ('twitterstatusfetcher.'.$this->_id); } + /** + * Run the daemon + * + * @return void + */ + function run() { + if (defined('SCRIPT_DEBUG')) { + common_debug($this->name() . + ': debugging log output enabled.'); + } + do { $flinks = $this->refreshFlinks(); - foreach ($flinks as $f){ + foreach ($flinks as $f) { // We have to disconnect from the DB before forking so // each sub-process will open its own connection and @@ -73,10 +116,11 @@ class TwitterStatusFetcher extends Daemon // Parent if (defined('SCRIPT_DEBUG')) { - common_debug("Parent: forked new status fetcher process " . $pid); + common_debug("Parent: forked new status ". + " fetcher process " . $pid); } - $this->children[] = $pid; + $this->_children[] = $pid; } else { @@ -86,41 +130,41 @@ class TwitterStatusFetcher extends Daemon } // Remove child from ps list as it finishes - while(($c = pcntl_wait($status, WNOHANG OR WUNTRACED)) > 0) { + while (($c = pcntl_wait($status, WNOHANG OR WUNTRACED)) > 0) { if (defined('SCRIPT_DEBUG')) { common_debug("Child $c finished."); } - $this->remove_ps($this->children, $c); + $this->removePs($this->_children, $c); } // Wait! We have too many damn kids. - if (sizeof($this->children) > MAXCHILDREN) { + if (sizeof($this->_children) > MAXCHILDREN) { if (defined('SCRIPT_DEBUG')) { common_debug('Too many children. Waiting...'); } - if (($c = pcntl_wait($status, WUNTRACED)) > 0){ + if (($c = pcntl_wait($status, WUNTRACED)) > 0) { if (defined('SCRIPT_DEBUG')) { common_debug("Finished waiting for $c"); } - $this->remove_ps($this->children, $c); + $this->removePs($this->_children, $c); } } } // Remove all children from the process list before restarting - while(($c = pcntl_wait($status, WUNTRACED)) > 0) { + while (($c = pcntl_wait($status, WUNTRACED)) > 0) { if (defined('SCRIPT_DEBUG')) { common_debug("Child $c finished."); } - $this->remove_ps($this->children, $c); + $this->removePs($this->_children, $c); } // Rest for a bit before we fetch more statuses @@ -137,10 +181,18 @@ class TwitterStatusFetcher extends Daemon } while (true); } - function refreshFlinks() { + /** + * Refresh the foreign links for this user + * + * @return void + */ + function refreshFlinks() + { $flink = new Foreign_link(); + $flink->service = 1; // Twitter + $flink->orderBy('last_noticesync'); $cnt = $flink->find(); @@ -166,7 +218,18 @@ class TwitterStatusFetcher extends Daemon return $flinks; } - function remove_ps(&$plist, $ps){ + /** + * Unknown + * + * @param array &$plist unknown. + * @param string $ps unknown. + * + * @return unknown + * @todo document + */ + + function removePs(&$plist, $ps) + { for ($i = 0; $i < sizeof($plist); $i++) { if ($plist[$i] == $ps) { unset($plist[$i]); @@ -178,7 +241,6 @@ class TwitterStatusFetcher extends Daemon function getTimeline($flink) { - if (empty($flink)) { common_log(LOG_WARNING, "Can't retrieve Foreign_link for foreign ID $fid"); @@ -247,64 +309,44 @@ class TwitterStatusFetcher extends Daemon return null; } + // XXX: change of screen name? + $uri = 'http://twitter.com/' . $status->user->screen_name . '/status/' . $status->id; $notice = Notice::staticGet('uri', $uri); // check to see if we've already imported the status + if (!$notice) { $notice = new Notice(); - $notice->profile_id = $id; - - $notice->query('BEGIN'); - - // XXX: figure out reply_to - $notice->reply_to = null; - - // XXX: Should this be common_sql_now() instead of status create date? - - $notice->created = strftime('%Y-%m-%d %H:%M:%S', - strtotime($status->created_at)); - $notice->content = $status->text; - $notice->rendered = common_render_content($status->text, $notice); - $notice->source = 'twitter'; - $notice->is_local = 0; - $notice->uri = $uri; - $notice_id = $notice->insert(); - - if (!$notice_id) { - common_log_db_error($notice, 'INSERT', __FILE__); - if (defined('SCRIPT_DEBUG')) { - common_debug('Could not save notice!'); - } - } - - // XXX: Figure out a better way to link Twitter replies? - $notice->saveReplies(); - - // XXX: Do we want to pollute our tag cloud with - // hashtags from Twitter? - $notice->saveTags(); - $notice->saveGroups(); - - $notice->query('COMMIT'); - - if (defined('SCRIPT_DEBUG')) { - common_debug("Saved status $status->id" . - " as notice $notice->id."); + $notice->profile_id = $id; + $notice->uri = $uri; + $notice->created = strftime('%Y-%m-%d %H:%M:%S', + strtotime($status->created_at)); + $notice->content = common_shorten_links($status->text); // XXX + $notice->rendered = common_render_content($notice->content, $notice); + $notice->source = 'twitter'; + $notice->reply_to = null; // XXX lookup reply + $notice->is_local = Notice::GATEWAY; + + if (Event::handle('StartNoticeSave', array(&$notice))) { + $id = $notice->insert(); + Event::handle('EndNoticeSave', array($notice)); } } - if (!Notice_inbox::staticGet('notice_id', $notice->id)) { - + if (!Notice_inbox::pkeyGet(array('notice_id' => $notice->id, + 'user_id' => $flink->user_id))) { // Add to inbox $inbox = new Notice_inbox(); - $inbox->user_id = $flink->user_id; + + $inbox->user_id = $flink->user_id; $inbox->notice_id = $notice->id; - $inbox->created = common_sql_now(); + $inbox->created = $notice->created; + $inbox->source = NOTICE_INBOX_SOURCE_GATEWAY; // From a private source $inbox->insert(); } @@ -379,12 +421,13 @@ class TwitterStatusFetcher extends Daemon } } - function checkAvatar($user, $profile) + function checkAvatar($twitter_user, $profile) { global $config; - $path_parts = pathinfo($user->profile_image_url); - $newname = 'Twitter_' . $user->id . '_' . + $path_parts = pathinfo($twitter_user->profile_image_url); + + $newname = 'Twitter_' . $twitter_user->id . '_' . $path_parts['basename']; $oldname = $profile->getAvatar(48)->filename; @@ -397,21 +440,56 @@ class TwitterStatusFetcher extends Daemon common_debug("old: $oldname new: $newname"); } - $img_root = substr($path_parts['basename'], 0, -11); - $ext = $path_parts['extension']; - $mediatype = $this->getMediatype($ext); + $this->updateAvatars($twitter_user, $profile); + } + + if ($this->missingAvatarFile($profile)) { - foreach (array('mini', 'normal', 'bigger') as $size) { - $url = $path_parts['dirname'] . '/' . - $img_root . '_' . $size . ".$ext"; - $filename = 'Twitter_' . $user->id . '_' . - $img_root . "_$size.$ext"; + if (defined('SCRIPT_DEBUG')) { + common_debug('Twitter user ' . $profile->nickname . + ' is missing one or more local avatars.'); + common_debug("old: $oldname new: $newname"); + } - if ($this->fetchAvatar($url, $filename)) { - $this->updateAvatar($profile->id, $size, $mediatype, $filename); - } + $this->updateAvatars($twitter_user, $profile); + } + + } + + function updateAvatars($twitter_user, $profile) { + + global $config; + + $path_parts = pathinfo($twitter_user->profile_image_url); + + $img_root = substr($path_parts['basename'], 0, -11); + $ext = $path_parts['extension']; + $mediatype = $this->getMediatype($ext); + + foreach (array('mini', 'normal', 'bigger') as $size) { + $url = $path_parts['dirname'] . '/' . + $img_root . '_' . $size . ".$ext"; + $filename = 'Twitter_' . $twitter_user->id . '_' . + $img_root . "_$size.$ext"; + + $this->updateAvatar($profile->id, $size, $mediatype, $filename); + $this->fetchAvatar($url, $filename); + } + } + + function missingAvatarFile($profile) { + + foreach (array(24, 48, 73) as $size) { + + $filename = $profile->getAvatar($size)->filename; + $avatarpath = Avatar::path($filename); + + if (file_exists($avatarpath) == FALSE) { + return true; } } + + return false; } function getMediatype($ext) @@ -464,7 +542,7 @@ class TwitterStatusFetcher extends Daemon $profile = Profile::staticGet($profile_id); - if (!$profile) { + if (empty($profile)) { if (defined('SCRIPT_DEBUG')) { common_debug("Couldn't get profile: $profile_id!"); } @@ -474,11 +552,8 @@ class TwitterStatusFetcher extends Daemon $sizes = array('mini' => 24, 'normal' => 48, 'bigger' => 73); $avatar = $profile->getAvatar($sizes[$size]); + // Delete the avatar, if present if ($avatar) { - if (defined('SCRIPT_DEBUG')) { - common_debug("Deleting $size avatar for $profile->nickname."); - } - @unlink(INSTALLDIR . '/avatar/' . $avatar->filename); $avatar->delete(); } @@ -523,7 +598,7 @@ class TwitterStatusFetcher extends Daemon $id = $avatar->insert(); - if (!$id) { + if (empty($id)) { common_log_db_error($avatar, 'INSERT', __FILE__); return null; } @@ -566,12 +641,22 @@ class TwitterStatusFetcher extends Daemon } } -ini_set("max_execution_time", "0"); -ini_set("max_input_time", "0"); -set_time_limit(0); -mb_internal_encoding('UTF-8'); declare(ticks = 1); -$fetcher = new TwitterStatusFetcher(); +if (have_option('i')) { + $id = get_option_value('i'); +} else if (have_option('--id')) { + $id = get_option_value('--id'); +} else if (count($args) > 0) { + $id = $args[0]; +} else { + $id = null; +} + +if (have_option('d') || have_option('debug')) { + define('SCRIPT_DEBUG', true); +} + +$fetcher = new TwitterStatusFetcher($id); $fetcher->runOnce();