4 * StatusNet - the distributed open-source microblogging tool
5 * Copyright (C) 2008, 2009, StatusNet, Inc.
7 * This program is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU Affero General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU Affero General Public License for more details.
17 * You should have received a copy of the GNU Affero General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
21 define('INSTALLDIR', realpath(dirname(__FILE__) . '/../../..'));
23 // Tune number of processes and how often to poll Twitter
24 // XXX: Should these things be in config.php?
25 define('MAXCHILDREN', 2);
26 define('POLL_INTERVAL', 60); // in seconds
28 $shortoptions = 'di::';
29 $longoptions = array('id::', 'debug');
31 $helptext = <<<END_OF_TRIM_HELP
32 Batch script for retrieving Twitter messages from foreign service.
34 -i --id Identity (default 'generic')
35 -d --debug Debug (lots of log output)
39 require_once INSTALLDIR . '/scripts/commandline.inc';
40 require_once INSTALLDIR . '/lib/daemon.php';
41 require_once INSTALLDIR . '/plugins/TwitterBridge/twitter.php';
44 * Fetcher for statuses from Twitter
46 * Fetches statuses from Twitter and inserts them as notices in local
51 * @author Zach Copley <zach@status.net>
52 * @author Evan Prodromou <evan@status.net>
53 * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
54 * @link http://status.net/
57 // NOTE: an Avatar path MUST be set in config.php for this
58 // script to work: e.g.: $config['avatar']['path'] = '/statusnet/avatar';
60 class TwitterStatusFetcher extends ParallelizingDaemon
65 * @param string $id the name/id of this daemon
66 * @param int $interval sleep this long before doing everything again
67 * @param int $max_children maximum number of child processes at a time
68 * @param boolean $debug debug output flag
73 function __construct($id = null, $interval = 60,
74 $max_children = 2, $debug = null)
76 parent::__construct($id, $interval, $max_children, $debug);
82 * @return string Name of the daemon.
87 return ('twitterstatusfetcher.'.$this->_id);
91 * Find all the Twitter foreign links for users who have requested
92 * importing of their friends' timelines
94 * @return array flinks an array of Foreign_link objects
99 global $_DB_DATAOBJECT;
101 $flink = new Foreign_link();
102 $conn = &$flink->getDatabaseConnection();
104 $flink->service = TWITTER_SERVICE;
105 $flink->orderBy('last_noticesync');
110 while ($flink->fetch()) {
112 if (($flink->noticesync & FOREIGN_NOTICE_RECV) ==
113 FOREIGN_NOTICE_RECV) {
114 $flinks[] = clone($flink);
122 unset($_DB_DATAOBJECT['CONNECTIONS']);
127 function childTask($flink) {
129 // Each child ps needs its own DB connection
131 // Note: DataObject::getDatabaseConnection() creates
132 // a new connection if there isn't one already
134 $conn = &$flink->getDatabaseConnection();
136 $this->getTimeline($flink);
138 $flink->last_friendsync = common_sql_now();
143 // XXX: Couldn't find a less brutal way to blow
144 // away a cached connection
146 global $_DB_DATAOBJECT;
147 unset($_DB_DATAOBJECT['CONNECTIONS']);
150 function getTimeline($flink)
153 common_log(LOG_WARNING, $this->name() .
154 " - Can't retrieve Foreign_link for foreign ID $fid");
158 common_debug($this->name() . ' - Trying to get timeline for Twitter user ' .
161 // XXX: Biggest remaining issue - How do we know at which status
162 // to start importing? How many statuses? Right now I'm going
163 // with the default last 20.
167 if (TwitterOAuthClient::isPackedToken($flink->credentials)) {
168 $token = TwitterOAuthClient::unpackToken($flink->credentials);
169 $client = new TwitterOAuthClient($token->key, $token->secret);
170 common_debug($this->name() . ' - Grabbing friends timeline with OAuth.');
172 $client = new TwitterBasicAuthClient($flink);
173 common_debug($this->name() . ' - Grabbing friends timeline with basic auth.');
179 $timeline = $client->statusesFriendsTimeline();
180 } catch (Exception $e) {
181 common_log(LOG_WARNING, $this->name() .
182 ' - Twitter client unable to get friends timeline for user ' .
183 $flink->user_id . ' - code: ' .
184 $e->getCode() . 'msg: ' . $e->getMessage());
187 if (empty($timeline)) {
188 common_log(LOG_WARNING, $this->name() . " - Empty timeline.");
192 // Reverse to preserve order
194 foreach (array_reverse($timeline) as $status) {
196 // Hacktastic: filter out stuff coming from this StatusNet
198 $source = mb_strtolower(common_config('integration', 'source'));
200 if (preg_match("/$source/", mb_strtolower($status->source))) {
201 common_debug($this->name() . ' - Skipping import of status ' .
202 $status->id . ' with source ' . $source);
206 $this->saveStatus($status, $flink);
209 // Okay, record the time we synced with Twitter for posterity
211 $flink->last_noticesync = common_sql_now();
215 function saveStatus($status, $flink)
217 $id = $this->ensureProfile($status->user);
219 $profile = Profile::staticGet($id);
221 if (empty($profile)) {
222 common_log(LOG_ERR, $this->name() .
223 ' - Problem saving notice. No associated Profile.');
227 // XXX: change of screen name?
229 $uri = 'http://twitter.com/' . $status->user->screen_name .
230 '/status/' . $status->id;
232 $notice = Notice::staticGet('uri', $uri);
234 // check to see if we've already imported the status
236 if (empty($notice)) {
238 $notice = new Notice();
240 $notice->profile_id = $id;
242 $notice->created = strftime('%Y-%m-%d %H:%M:%S',
243 strtotime($status->created_at));
244 $notice->content = common_shorten_links($status->text); // XXX
245 $notice->rendered = common_render_content($notice->content, $notice);
246 $notice->source = 'twitter';
247 $notice->reply_to = null; // XXX: lookup reply
248 $notice->is_local = Notice::GATEWAY;
250 if (Event::handle('StartNoticeSave', array(&$notice))) {
251 $id = $notice->insert();
252 Event::handle('EndNoticeSave', array($notice));
256 if (!Notice_inbox::pkeyGet(array('notice_id' => $notice->id,
257 'user_id' => $flink->user_id))) {
259 $inbox = new Notice_inbox();
261 $inbox->user_id = $flink->user_id;
262 $inbox->notice_id = $notice->id;
263 $inbox->created = $notice->created;
264 $inbox->source = NOTICE_INBOX_SOURCE_GATEWAY; // From a private source
270 function ensureProfile($user)
272 // check to see if there's already a profile for this user
274 $profileurl = 'http://twitter.com/' . $user->screen_name;
275 $profile = Profile::staticGet('profileurl', $profileurl);
277 if (!empty($profile)) {
278 common_debug($this->name() .
279 " - Profile for $profile->nickname found.");
281 // Check to see if the user's Avatar has changed
283 $this->checkAvatar($user, $profile);
287 common_debug($this->name() . ' - Adding profile and remote profile ' .
288 "for Twitter user: $profileurl.");
290 $profile = new Profile();
291 $profile->query("BEGIN");
293 $profile->nickname = $user->screen_name;
294 $profile->fullname = $user->name;
295 $profile->homepage = $user->url;
296 $profile->bio = $user->description;
297 $profile->location = $user->location;
298 $profile->profileurl = $profileurl;
299 $profile->created = common_sql_now();
301 $id = $profile->insert();
304 common_log_db_error($profile, 'INSERT', __FILE__);
305 $profile->query("ROLLBACK");
309 // check for remote profile
311 $remote_pro = Remote_profile::staticGet('uri', $profileurl);
313 if (empty($remote_pro)) {
315 $remote_pro = new Remote_profile();
317 $remote_pro->id = $id;
318 $remote_pro->uri = $profileurl;
319 $remote_pro->created = common_sql_now();
321 $rid = $remote_pro->insert();
324 common_log_db_error($profile, 'INSERT', __FILE__);
325 $profile->query("ROLLBACK");
330 $profile->query("COMMIT");
332 $this->saveAvatars($user, $id);
338 function checkAvatar($twitter_user, $profile)
342 $path_parts = pathinfo($twitter_user->profile_image_url);
344 $newname = 'Twitter_' . $twitter_user->id . '_' .
345 $path_parts['basename'];
347 $oldname = $profile->getAvatar(48)->filename;
349 if ($newname != $oldname) {
350 common_debug($this->name() . ' - Avatar for Twitter user ' .
351 "$profile->nickname has changed.");
352 common_debug($this->name() . " - old: $oldname new: $newname");
354 $this->updateAvatars($twitter_user, $profile);
357 if ($this->missingAvatarFile($profile)) {
358 common_debug($this->name() . ' - Twitter user ' .
360 ' is missing one or more local avatars.');
361 common_debug($this->name() ." - old: $oldname new: $newname");
363 $this->updateAvatars($twitter_user, $profile);
368 function updateAvatars($twitter_user, $profile) {
372 $path_parts = pathinfo($twitter_user->profile_image_url);
374 $img_root = substr($path_parts['basename'], 0, -11);
375 $ext = $path_parts['extension'];
376 $mediatype = $this->getMediatype($ext);
378 foreach (array('mini', 'normal', 'bigger') as $size) {
379 $url = $path_parts['dirname'] . '/' .
380 $img_root . '_' . $size . ".$ext";
381 $filename = 'Twitter_' . $twitter_user->id . '_' .
382 $img_root . "_$size.$ext";
384 $this->updateAvatar($profile->id, $size, $mediatype, $filename);
385 $this->fetchAvatar($url, $filename);
389 function missingAvatarFile($profile) {
391 foreach (array(24, 48, 73) as $size) {
393 $filename = $profile->getAvatar($size)->filename;
394 $avatarpath = Avatar::path($filename);
396 if (file_exists($avatarpath) == FALSE) {
404 function getMediatype($ext)
408 switch (strtolower($ext)) {
410 $mediatype = 'image/jpg';
413 $mediatype = 'image/gif';
416 $mediatype = 'image/png';
422 function saveAvatars($user, $id)
426 $path_parts = pathinfo($user->profile_image_url);
427 $ext = $path_parts['extension'];
428 $end = strlen('_normal' . $ext);
429 $img_root = substr($path_parts['basename'], 0, -($end+1));
430 $mediatype = $this->getMediatype($ext);
432 foreach (array('mini', 'normal', 'bigger') as $size) {
433 $url = $path_parts['dirname'] . '/' .
434 $img_root . '_' . $size . ".$ext";
435 $filename = 'Twitter_' . $user->id . '_' .
436 $img_root . "_$size.$ext";
438 if ($this->fetchAvatar($url, $filename)) {
439 $this->newAvatar($id, $size, $mediatype, $filename);
441 common_log(LOG_WARNING, $this->id() .
442 " - Problem fetching Avatar: $url");
447 function updateAvatar($profile_id, $size, $mediatype, $filename) {
449 common_debug($this->name() . " - Updating avatar: $size");
451 $profile = Profile::staticGet($profile_id);
453 if (empty($profile)) {
454 common_debug($this->name() . " - Couldn't get profile: $profile_id!");
458 $sizes = array('mini' => 24, 'normal' => 48, 'bigger' => 73);
459 $avatar = $profile->getAvatar($sizes[$size]);
461 // Delete the avatar, if present
467 $this->newAvatar($profile->id, $size, $mediatype, $filename);
470 function newAvatar($profile_id, $size, $mediatype, $filename)
474 $avatar = new Avatar();
475 $avatar->profile_id = $profile_id;
480 $avatar->height = 24;
484 $avatar->height = 48;
488 // Note: Twitter's big avatars are a different size than
489 // StatusNet's (StatusNet's = 96)
492 $avatar->height = 73;
495 $avatar->original = 0; // we don't have the original
496 $avatar->mediatype = $mediatype;
497 $avatar->filename = $filename;
498 $avatar->url = Avatar::url($filename);
500 common_debug($this->name() . " - New filename: $avatar->url");
502 $avatar->created = common_sql_now();
504 $id = $avatar->insert();
507 common_log_db_error($avatar, 'INSERT', __FILE__);
511 common_debug($this->name() .
512 " - Saved new $size avatar for $profile_id.");
517 function fetchAvatar($url, $filename)
519 $avatar_dir = INSTALLDIR . '/avatar/';
521 $avatarfile = $avatar_dir . $filename;
523 $out = fopen($avatarfile, 'wb');
525 common_log(LOG_WARNING, $this->name() .
526 " - Couldn't open file $filename");
530 common_debug($this->name() . " - Fetching Twitter avatar: $url");
533 curl_setopt($ch, CURLOPT_URL, $url);
534 curl_setopt($ch, CURLOPT_FILE, $out);
535 curl_setopt($ch, CURLOPT_BINARYTRANSFER, true);
536 curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
537 curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 0);
538 $result = curl_exec($ch);
550 if (have_option('i')) {
551 $id = get_option_value('i');
552 } else if (have_option('--id')) {
553 $id = get_option_value('--id');
554 } else if (count($args) > 0) {
560 if (have_option('d') || have_option('debug')) {
564 $fetcher = new TwitterStatusFetcher($id, 60, 2, $debug);