3 * Name: Retrieve Feed Content
4 * Description: Follow the permalink of RSS/Atom feed items and replace the summary with the full content.
6 * Author: Matthew Exon <http://mat.exon.name>
10 require_once('include/html2bbcode.php');
11 require_once('include/Photo.php');
13 function retriever_install() {
14 register_hook('plugin_settings', 'addon/retriever/retriever.php', 'retriever_plugin_settings');
15 register_hook('plugin_settings_post', 'addon/retriever/retriever.php', 'retriever_plugin_settings_post');
16 register_hook('post_remote', 'addon/retriever/retriever.php', 'retriever_post_remote_hook');
17 register_hook('contact_photo_menu', 'addon/retriever/retriever.php', 'retriever_contact_photo_menu');
18 register_hook('cron', 'addon/retriever/retriever.php', 'retriever_cron');
20 $r = q("SELECT `id` FROM `pconfig` WHERE `cat` LIKE 'retriever_%%'");
21 if (count($r) || (get_config('retriever', 'dbversion') == '0.1')) {
22 $retrievers = array();
23 $r = q("SELECT SUBSTRING(`cat`, 10) AS `contact`, `k`, `v` FROM `pconfig` WHERE `cat` LIKE 'retriever%%'");
25 $retrievers[$rr['contact']][$rr['k']] = $rr['v'];
27 foreach ($retrievers as $k => $v) {
28 $rr = q("SELECT `uid` FROM `contact` WHERE `id` = %d", intval($k));
31 q("INSERT INTO `retriever_rule` (`uid`, `contact-id`, `data`) VALUES (%d, %d, '%s')",
32 intval($uid), intval($k), dbesc(json_encode($v)));
34 q("DELETE FROM `pconfig` WHERE `cat` LIKE 'retriever_%%'");
35 set_config('retriever', 'dbversion', '0.2');
37 if (get_config('retriever', 'dbversion') == '0.2') {
38 q("ALTER TABLE `retriever_resource` DROP COLUMN `retriever`");
39 set_config('retriever', 'dbversion', '0.3');
41 if (get_config('retriever', 'dbversion') == '0.3') {
42 q("ALTER TABLE `retriever_item` MODIFY COLUMN `item-uri` varchar(800) CHARACTER SET ascii NOT NULL");
43 q("ALTER TABLE `retriever_resource` MODIFY COLUMN `url` varchar(800) CHARACTER SET ascii NOT NULL");
44 set_config('retriever', 'dbversion', '0.4');
46 if (get_config('retriever', 'dbversion') == '0.4') {
47 q("ALTER TABLE `retriever_item` ADD COLUMN `finished` tinyint(1) unsigned NOT NULL DEFAULT '0'");
48 set_config('retriever', 'dbversion', '0.5');
50 if (get_config('retriever', 'dbversion') == '0.5') {
51 q('ALTER TABLE `retriever_resource` CHANGE `created` `created` timestamp NOT NULL DEFAULT now()');
52 q('ALTER TABLE `retriever_resource` CHANGE `completed` `completed` timestamp NULL DEFAULT NULL');
53 q('ALTER TABLE `retriever_resource` CHANGE `last-try` `last-try` timestamp NULL DEFAULT NULL');
54 q('ALTER TABLE `retriever_item` DROP KEY `all`');
55 q('ALTER TABLE `retriever_item` ADD KEY `all` (`item-uri`, `item-uid`, `contact-id`)');
56 set_config('retriever', 'dbversion', '0.6');
58 if (get_config('retriever', 'dbversion') == '0.6') {
59 q('ALTER TABLE `retriever_item` CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin');
60 q('ALTER TABLE `retriever_item` CHANGE `item-uri` `item-uri` varchar(800) CHARACTER SET ascii COLLATE ascii_bin NOT NULL');
61 q('ALTER TABLE `retriever_resource` CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin');
62 q('ALTER TABLE `retriever_resource` CHANGE `url` `url` varchar(800) CHARACTER SET ascii COLLATE ascii_bin NOT NULL');
63 q('ALTER TABLE `retriever_rule` CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin');
64 set_config('retriever', 'dbversion', '0.7');
66 if (get_config('retriever', 'dbversion') == '0.7') {
67 $r = q("SELECT `id`, `data` FROM `retriever_rule`");
69 logger('retriever_install: retriever ' . $rr['id'] . ' old config ' . $rr['data'], LOGGER_DATA);
70 $data = json_decode($rr['data'], true);
71 if ($data['pattern']) {
73 if (preg_match("/\/(.*)\//", $data['pattern'], $matches)) {
74 $data['pattern'] = $matches[1];
79 foreach (explode('|', $data['match']) as $component) {
81 if (preg_match("/([A-Za-z][A-Za-z0-9]*)\[@([A-Za-z][a-z0-9]*)='([^']*)'\]/", $component, $matches)) {
83 'element' => $matches[1],
84 'attribute' => $matches[2],
85 'value' => $matches[3]);
87 if (preg_match("/([A-Za-z][A-Za-z0-9]*)\[contains(concat(' ',normalize-space(@class),' '),' ([^ ']+) ')]/", $component, $matches)) {
89 'element' => $matches[1],
90 'attribute' => $matches[2],
91 'value' => $matches[3]);
94 $data['include'] = $include;
95 unset($data['match']);
97 if ($data['remove']) {
99 foreach (explode('|', $data['remove']) as $component) {
101 if (preg_match("/([A-Za-z][A-Za-z0-9]*)\[@([A-Za-z][a-z0-9]*)='([^']*)'\]/", $component, $matches)) {
103 'element' => $matches[1],
104 'attribute' => $matches[2],
105 'value' => $matches[3]);
107 if (preg_match("/([A-Za-z][A-Za-z0-9]*)\[contains(concat(' ',normalize-space(@class),' '),' ([^ ']+) ')]/", $component, $matches)) {
109 'element' => $matches[1],
110 'attribute' => $matches[2],
111 'value' => $matches[3]);
114 $data['exclude'] = $exclude;
115 unset($data['remove']);
117 $r = q('UPDATE `retriever_rule` SET `data` = "%s" WHERE `id` = %d', dbesc(json_encode($data)), $rr['id']);
118 logger('retriever_install: retriever ' . $rr['id'] . ' new config ' . json_encode($data), LOGGER_DATA);
120 set_config('retriever', 'dbversion', '0.8');
122 if (get_config('retriever', 'dbversion') == '0.8') {
123 q("ALTER TABLE `retriever_resource` ADD COLUMN `http-code` smallint(1) unsigned NULL DEFAULT NULL");
124 set_config('retriever', 'dbversion', '0.9');
126 if (get_config('retriever', 'dbversion') == '0.9') {
127 q("ALTER TABLE `retriever_item` DROP COLUMN `parent`");
128 q("ALTER TABLE `retriever_resource` ADD COLUMN `redirect-url` varchar(800) CHARACTER SET ascii COLLATE ascii_bin NULL DEFAULT NULL");
129 set_config('retriever', 'dbversion', '0.10');
132 if (get_config('retriever', 'dbversion') != '0.10') {
133 $schema = file_get_contents(dirname(__file__).'/database.sql');
134 $arr = explode(';', $schema);
135 foreach ($arr as $a) {
138 set_config('retriever', 'dbversion', '0.10');
142 function retriever_uninstall() {
143 unregister_hook('plugin_settings', 'addon/retriever/retriever.php', 'retriever_plugin_settings');
144 unregister_hook('plugin_settings_post', 'addon/retriever/retriever.php', 'retriever_plugin_settings_post');
145 unregister_hook('post_remote', 'addon/retriever/retriever.php', 'retriever_post_remote_hook');
146 unregister_hook('plugin_settings', 'addon/retriever/retriever.php', 'retriever_plugin_settings');
147 unregister_hook('plugin_settings_post', 'addon/retriever/retriever.php', 'retriever_plugin_settings_post');
148 unregister_hook('contact_photo_menu', 'addon/retriever/retriever.php', 'retriever_contact_photo_menu');
149 unregister_hook('cron', 'addon/retriever/retriever.php', 'retriever_cron');
152 function retriever_module() {}
154 function retriever_cron($a, $b) {
155 // 100 is a nice sane number. Maybe this should be configurable.
156 retriever_retrieve_items(100);
160 $retriever_item_count = 0;
162 function retriever_retrieve_items($max_items) {
163 global $retriever_item_count;
165 $retriever_schedule = array(array(1,'minute'),
173 $schedule_clauses = array();
174 for ($i = 0; $i < count($retriever_schedule); $i++) {
175 $num = $retriever_schedule[$i][0];
176 $unit = $retriever_schedule[$i][1];
177 array_push($schedule_clauses,
178 '(`num-tries` = ' . $i . ' AND TIMESTAMPADD(' . dbesc($unit) .
179 ', ' . intval($num) . ', `last-try`) < now())');
182 $retrieve_items = $max_items - $retriever_item_count;
183 logger('retriever_retrieve_items: asked for maximum ' . $max_items . ', already retrieved ' . $retriever_item_count . ', retrieve ' . $retrieve_items, LOGGER_DEBUG);
185 $r = q("SELECT * FROM `retriever_resource` WHERE `completed` IS NULL AND (`last-try` IS NULL OR %s) ORDER BY `last-try` ASC LIMIT %d",
186 dbesc(implode($schedule_clauses, ' OR ')),
187 intval($retrieve_items));
191 if (count($r) == 0) {
194 logger('retriever_retrieve_items: found ' . count($r) . ' waiting resources in database', LOGGER_DEBUG);
195 foreach ($r as $rr) {
196 retrieve_resource($rr);
197 $retriever_item_count++;
199 $retrieve_items = $max_items - $retriever_item_count;
201 while ($retrieve_items > 0);
203 /* Look for items that are waiting even though the resource has
204 * completed. This usually happens because we've been asked to
205 * retrospectively apply a config change. It could also happen
206 * due to a cron job dying or something. */
207 $r = q("SELECT retriever_resource.`id` as resource, retriever_item.`id` as item FROM retriever_resource, retriever_item, retriever_rule WHERE retriever_item.`finished` = 0 AND retriever_item.`resource` = retriever_resource.`id` AND retriever_resource.`completed` IS NOT NULL AND retriever_item.`contact-id` = retriever_rule.`contact-id` AND retriever_item.`item-uid` = retriever_rule.`uid` LIMIT %d",
208 intval($retrieve_items));
212 logger('retriever_retrieve_items: items waiting even though resource has completed: ' . count($r), LOGGER_DEBUG);
213 foreach ($r as $rr) {
214 $resource = q("SELECT * FROM retriever_resource WHERE `id` = %d", $rr['resource']);
215 $retriever_item = retriever_get_retriever_item($rr['item']);
216 if (!$retriever_item) {
217 logger('retriever_retrieve_items: no retriever item with id ' . $rr['item'], LOGGER_NORMAL);
220 $item = retriever_get_item($retriever_item);
222 logger('retriever_retrieve_items: no item ' . $retriever_item['item-uri'], LOGGER_NORMAL);
225 $retriever = get_retriever($item['contact-id'], $item['uid']);
227 logger('retriever_retrieve_items: no retriever for item ' .
228 $retriever_item['item-uri'] . ' ' . $retriever_item['uid'] . ' ' . $item['contact-id'],
232 retriever_apply_completed_resource_to_item($retriever, $item, $resource[0]);
233 q("UPDATE `retriever_item` SET `finished` = 1 WHERE id = %d",
234 intval($retriever_item['id']));
235 retriever_check_item_completed($item);
239 function retriever_tidy() {
240 q("DELETE FROM retriever_resource WHERE completed IS NOT NULL AND completed < DATE_SUB(now(), INTERVAL 1 WEEK)");
241 q("DELETE FROM retriever_resource WHERE completed IS NULL AND created < DATE_SUB(now(), INTERVAL 3 MONTH)");
243 $r = q("SELECT retriever_item.id FROM retriever_item LEFT OUTER JOIN retriever_resource ON (retriever_item.resource = retriever_resource.id) WHERE retriever_resource.id is null");
244 logger('retriever_tidy: found ' . count($r) . ' retriever_items with no retriever_resource');
245 foreach ($r as $rr) {
246 q('DELETE FROM retriever_item WHERE id = %d', intval($rr['id']));
250 function retrieve_resource($resource) {
253 logger('retrieve_resource: ' . ($resource['num-tries'] + 1) .
254 ' attempt at resource ' . $resource['id'] . ' ' . $resource['url'], LOGGER_DEBUG);
256 $cookiejar = tempnam(get_temppath(), 'cookiejar-retriever-');
257 $fetch_result = z_fetch_url($resource['url'], $resource['binary'], $redirects, array('cookiejar' => $cookiejar));
259 $resource['data'] = $fetch_result['body'];
260 $resource['http-code'] = $a->get_curl_code();
261 $resource['type'] = $a->get_curl_content_type();
262 $resource['redirect-url'] = $fetch_result['redirect_url'];
263 logger('retrieve_resource: got code ' . $resource['http-code'] .
264 ' retrieving resource ' . $resource['id'] .
265 ' final url ' . $resource['redirect-url'], LOGGER_DEBUG);
266 q("UPDATE `retriever_resource` SET `last-try` = now(), `num-tries` = `num-tries` + 1, `http-code` = %d, `redirect-url` = '%s' WHERE id = %d",
267 intval($resource['http-code']),
268 dbesc($resource['redirect-url']),
269 intval($resource['id']));
270 if ($resource['data']) {
271 q("UPDATE `retriever_resource` SET `completed` = now(), `data` = '%s', `type` = '%s' WHERE id = %d",
272 dbesc($resource['data']),
273 dbesc($resource['type']),
274 intval($resource['id']));
275 retriever_resource_completed($resource);
279 function get_retriever($contact_id, $uid, $create = false) {
280 $r = q("SELECT * FROM `retriever_rule` WHERE `contact-id` = %d AND `uid` = %d",
281 intval($contact_id), intval($uid));
283 $r[0]['data'] = json_decode($r[0]['data'], true);
287 q("INSERT INTO `retriever_rule` (`uid`, `contact-id`) VALUES (%d, %d)",
288 intval($uid), intval($contact_id));
289 $r = q("SELECT * FROM `retriever_rule` WHERE `contact-id` = %d AND `uid` = %d",
290 intval($contact_id), intval($uid));
295 function retriever_get_retriever_item($id) {
296 $retriever_items = q("SELECT * FROM `retriever_item` WHERE id = %d", intval($id));
297 if (count($retriever_items) != 1) {
298 logger('retriever_get_retriever_item: unable to find retriever_item ' . $id, LOGGER_NORMAL);
301 return $retriever_items[0];
304 function retriever_get_item($retriever_item) {
305 $items = q("SELECT * FROM `item` WHERE `uri` = '%s' AND `uid` = %d AND `contact-id` = %d",
306 dbesc($retriever_item['item-uri']),
307 intval($retriever_item['item-uid']),
308 intval($retriever_item['contact-id']));
309 if (count($items) != 1) {
310 logger('retriever_get_item: unexpected number of results ' .
311 count($items) . " when searching for item $uri $uid $cid", LOGGER_NORMAL);
317 function retriever_item_completed($retriever_item_id, $resource) {
318 logger('retriever_item_completed: id ' . $retriever_item_id . ' url ' . $resource['url'], LOGGER_DEBUG);
320 $retriever_item = retriever_get_retriever_item($retriever_item_id);
321 if (!$retriever_item) {
324 // Note: the retriever might be null. Doesn't matter.
325 $retriever = get_retriever($retriever_item['contact-id'], $retriever_item['item-uid']);
326 $item = retriever_get_item($retriever_item);
331 retriever_apply_completed_resource_to_item($retriever, $item, $resource);
333 q("UPDATE `retriever_item` SET `finished` = 1 WHERE id = %d",
334 intval($retriever_item['id']));
335 retriever_check_item_completed($item);
338 function retriever_resource_completed($resource) {
339 logger('retriever_resource_completed: id ' . $resource['id'] . ' url ' . $resource['url'], LOGGER_DEBUG);
340 $r = q("SELECT `id` FROM `retriever_item` WHERE `resource` = %d", $resource['id']);
341 foreach ($r as $rr) {
342 retriever_item_completed($rr['id'], $resource);
346 function apply_retrospective($retriever, $num) {
347 $r = q("SELECT * FROM `item` WHERE `contact-id` = %d ORDER BY `received` DESC LIMIT %d",
348 intval($retriever['contact-id']), intval($num));
349 foreach ($r as $item) {
350 q('UPDATE `item` SET `visible` = 0 WHERE `id` = %d', $item['id']);
351 q('UPDATE `thread` SET `visible` = 0 WHERE `iid` = %d', $item['id']);
352 retriever_on_item_insert($retriever, $item);
356 function retriever_on_item_insert($retriever, &$item) {
357 if (!$retriever || !$retriever['id']) {
358 logger('retriever_on_item_insert: No retriever supplied', LOGGER_NORMAL);
361 if (!$retriever["data"]['enable'] == "on") {
364 if ($retriever["data"]['pattern']) {
365 $url = preg_replace('/' . $retriever["data"]['pattern'] . '/', $retriever["data"]['replace'], $item['plink']);
366 logger('retriever_on_item_insert: Changed ' . $item['plink'] . ' to ' . $url, LOGGER_DATA);
369 $url = $item['plink'];
372 $resource = add_retriever_resource($url);
373 $retriever_item_id = add_retriever_item($item, $resource);
376 function add_retriever_resource($url, $binary = false) {
377 logger('add_retriever_resource: ' . $url, LOGGER_DEBUG);
379 $scheme = parse_url($url, PHP_URL_SCHEME);
380 if ($scheme == 'data') {
381 $fp = fopen($url, 'r');
382 $meta = stream_get_meta_data($fp);
383 $type = $meta['mediatype'];
384 $data = stream_get_contents($fp);
387 $url = 'md5://' . hash('md5', $url);
388 $r = q("SELECT * FROM `retriever_resource` WHERE `url` = '%s'", dbesc($url));
391 logger('add_retriever_resource: Resource ' . $url . ' already requested', LOGGER_DEBUG);
395 logger('retrieve_resource: got data URL type ' . $resource['type'], LOGGER_DEBUG);
396 q("INSERT INTO `retriever_resource` (`type`, `binary`, `url`, `completed`, `data`) " .
397 "VALUES ('%s', %d, '%s', now(), '%s')",
399 intval($binary ? 1 : 0),
402 $r = q("SELECT * FROM `retriever_resource` WHERE `url` = '%s'", dbesc($url));
405 retriever_resource_completed($resource);
410 if (strlen($url) > 800) {
411 logger('add_retriever_resource: URL is longer than 800 characters', LOGGER_NORMAL);
414 $r = q("SELECT * FROM `retriever_resource` WHERE `url` = '%s'", dbesc($url));
417 logger('add_retriever_resource: Resource ' . $url . ' already requested', LOGGER_DEBUG);
421 q("INSERT INTO `retriever_resource` (`binary`, `url`) " .
422 "VALUES (%d, '%s')", intval($binary ? 1 : 0), dbesc($url));
423 $r = q("SELECT * FROM `retriever_resource` WHERE `url` = '%s'", dbesc($url));
427 function add_retriever_item(&$item, $resource) {
428 logger('add_retriever_item: ' . $resource['url'] . ' for ' . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id'], LOGGER_DEBUG);
430 q("INSERT INTO `retriever_item` (`item-uri`, `item-uid`, `contact-id`, `resource`) " .
431 "VALUES ('%s', %d, %d, %d)",
432 dbesc($item['uri']), intval($item['uid']), intval($item['contact-id']), intval($resource["id"]));
433 $r = q("SELECT id FROM `retriever_item` WHERE " .
434 "`item-uri` = '%s' AND `item-uid` = %d AND `contact-id` = %d AND `resource` = %d ORDER BY id DESC",
435 dbesc($item['uri']), intval($item['uid']), intval($item['contact-id']), intval($resource['id']));
437 logger("add_retriever_item: couldn't create retriever item for " .
438 $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id'],
442 logger('add_retriever_item: created retriever_item ' . $r[0]['id'] . ' for item ' . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id'], LOGGER_DEBUG);
446 function retriever_get_encoding($resource) {
448 if (preg_match('/charset=(.*)/', $resource['type'], $matches)) {
449 return trim(array_pop($matches));
454 function retriever_apply_xslt_text($xslt_text, $doc) {
456 logger('retriever_apply_xslt_text: empty XSLT text', LOGGER_NORMAL);
459 $xslt_doc = new DOMDocument();
460 if (!$xslt_doc->loadXML($xslt_text)) {
461 logger('retriever_apply_xslt_text: could not load XML', LOGGER_NORMAL);
464 $xp = new XsltProcessor();
465 $xp->importStylesheet($xslt_doc);
466 $result = $xp->transformToDoc($doc);
470 function retriever_apply_dom_filter($retriever, &$item, $resource) {
471 logger('retriever_apply_dom_filter: applying XSLT to ' . $item['id'] . ' ' . $item['uri'] . ' contact ' . $item['contact-id'], LOGGER_DEBUG);
473 if (!$retriever['data']['include'] && !$retriever['data']['customxslt']) {
476 if (!$resource['data']) {
477 logger('retriever_apply_dom_filter: no text to work with', LOGGER_NORMAL);
481 $encoding = retriever_get_encoding($resource);
482 $content = mb_convert_encoding($resource['data'], 'HTML-ENTITIES', $encoding);
483 $doc = new DOMDocument('1.0', 'UTF-8');
484 if (strpos($resource['type'], 'html') !== false) {
485 @$doc->loadHTML($content);
488 $doc->loadXML($content);
491 $params = array('$spec' => $retriever['data']);
492 $extract_template = get_markup_template('extract.tpl', 'addon/retriever/');
493 $extract_xslt = replace_macros($extract_template, $params);
494 if ($retriever['data']['include']) {
495 $doc = retriever_apply_xslt_text($extract_xslt, $doc);
497 if ($retriever['data']['customxslt']) {
498 $doc = retriever_apply_xslt_text($retriever['data']['customxslt'], $doc);
501 logger('retriever_apply_dom_filter: failed to apply extract XSLT template', LOGGER_NORMAL);
505 $components = parse_url($resource['redirect-url']);
506 $rooturl = $components['scheme'] . "://" . $components['host'];
507 $dirurl = $rooturl . dirname($components['path']) . "/";
508 $params = array('$dirurl' => $dirurl, '$rooturl' => $rooturl);
509 $fix_urls_template = get_markup_template('fix-urls.tpl', 'addon/retriever/');
510 $fix_urls_xslt = replace_macros($fix_urls_template, $params);
511 $doc = retriever_apply_xslt_text($fix_urls_xslt, $doc);
513 logger('retriever_apply_dom_filter: failed to apply fix urls XSLT template', LOGGER_NORMAL);
517 $item['body'] = html2bbcode($doc->saveXML());
518 if (!strlen($item['body'])) {
519 logger('retriever_apply_dom_filter retriever ' . $retriever['id'] . ' item ' . $item['id'] . ': output was empty', LOGGER_NORMAL);
522 $item['body'] .= "\n\n" . t('Retrieved') . ' ' . date("Y-m-d") . ': [url=';
523 $item['body'] .= $item['plink'];
524 $item['body'] .= ']' . $item['plink'] . '[/url]';
525 q("UPDATE `item` SET `body` = '%s' WHERE `id` = %d",
526 dbesc($item['body']), intval($item['id']));
529 function retrieve_images(&$item) {
531 preg_match_all("/\[img\=([0-9]*)x([0-9]*)\](.*?)\[\/img\]/ism", $item["body"], $matches1);
533 preg_match_all("/\[img\](.*?)\[\/img\]/ism", $item["body"], $matches2);
534 $matches = array_merge($matches1[3], $matches2[1]);
535 logger('retrieve_images: found ' . count($matches) . ' images for item ' . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id'], LOGGER_DEBUG);
536 foreach ($matches as $url) {
537 if (strpos($url, get_app()->get_baseurl()) === FALSE) {
538 $resource = add_retriever_resource($url, true);
539 if (!$resource['completed']) {
540 add_retriever_item($item, $resource);
543 retriever_transform_images($item, $resource);
549 function retriever_check_item_completed(&$item)
551 $r = q('SELECT count(*) FROM retriever_item WHERE `item-uri` = "%s" ' .
552 'AND `item-uid` = %d AND `contact-id` = %d AND `finished` = 0',
553 dbesc($item['uri']), intval($item['uid']),
554 intval($item['contact-id']));
555 $waiting = $r[0]['count(*)'];
556 logger('retriever_check_item_completed: item ' . $item['uri'] . ' ' . $item['uid']
557 . ' '. $item['contact-id'] . ' waiting for ' . $waiting . ' resources', LOGGER_DEBUG);
558 $old_visible = $item['visible'];
559 $item['visible'] = $waiting ? 0 : 1;
560 if (($item['id'] > 0) && ($old_visible != $item['visible'])) {
561 logger('retriever_check_item_completed: changing visible flag to ' . $item['visible'] . ' and invoking notifier ("edit_post", ' . $item['id'] . ')', LOGGER_DEBUG);
562 q("UPDATE `item` SET `visible` = %d WHERE `id` = %d",
563 intval($item['visible']),
564 intval($item['id']));
565 q("UPDATE `thread` SET `visible` = %d WHERE `iid` = %d",
566 intval($item['visible']),
567 intval($item['id']));
571 function retriever_apply_completed_resource_to_item($retriever, &$item, $resource) {
572 logger('retriever_apply_completed_resource_to_item: retriever ' .
573 ($retriever ? $retriever['id'] : 'none') .
574 ' resource ' . $resource['url'] . ' plink ' . $item['plink'], LOGGER_DEBUG);
575 if (strpos($resource['type'], 'image') !== false) {
576 retriever_transform_images($item, $resource);
581 if ((strpos($resource['type'], 'html') !== false) ||
582 (strpos($resource['type'], 'xml') !== false)) {
583 retriever_apply_dom_filter($retriever, $item, $resource);
584 if ($retriever["data"]['images'] ) {
585 retrieve_images($item);
590 function retriever_store_photo($item, &$resource) {
591 $hash = photo_new_resource();
593 if (class_exists('Imagick')) {
595 $image = new Imagick();
596 $image->readImageBlob($resource['data']);
597 $resource['width'] = $image->getImageWidth();
598 $resource['height'] = $image->getImageHeight();
600 catch (Exception $e) {
601 logger("ImageMagick couldn't process image " . $resource['id'] . " " . $resource['url'] . ' length ' . strlen($resource['data']) . ': ' . $e->getMessage(), LOGGER_DEBUG);
605 if (!array_key_exists('width', $resource)) {
606 $image = @imagecreatefromstring($resource['data']);
607 if ($image === false) {
608 logger("Couldn't process image " . $resource['id'] . " " . $resource['url'], LOGGER_DEBUG);
611 $resource['width'] = imagesx($image);
612 $resource['height'] = imagesy($image);
613 imagedestroy($image);
616 $url_components = parse_url($resource['url']);
617 $filename = basename($url_components['path']);
618 if (!strlen($filename)) {
621 $r = q("INSERT INTO `photo`
622 ( `uid`, `contact-id`, `guid`, `resource-id`, `created`, `edited`, `filename`, `type`, `album`, `height`, `width`, `datasize`, `data` )
623 VALUES ( %d, %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, %d, %d, '%s' )",
624 intval($item['item-uid']),
625 intval($item['contact-id']),
628 dbesc(datetime_convert()),
629 dbesc(datetime_convert()),
631 dbesc($resource['type']),
632 dbesc('Retrieved Images'),
633 intval($resource['height']),
634 intval($resource['width']),
635 intval(strlen($resource['data'])),
636 dbesc($resource['data'])
642 function retriever_transform_images(&$item, $resource) {
643 if (!$resource["data"]) {
644 logger('retriever_transform_images: no data available for '
645 . $resource['id'] . ' ' . $resource['url'], LOGGER_NORMAL);
649 $hash = retriever_store_photo($item, $resource);
650 if ($hash === false) {
651 logger('retriever_transform_images: unable to store photo '
652 . $resource['id'] . ' ' . $resource['url'], LOGGER_NORMAL);
656 $new_url = get_app()->get_baseurl() . '/photo/' . $hash;
657 logger('retriever_transform_images: replacing ' . $resource['url'] . ' with ' .
658 $new_url . ' in item ' . $item['plink'], LOGGER_DEBUG);
659 $transformed = str_replace($resource["url"], $new_url, $item['body']);
660 if ($transformed === $item['body']) {
664 $item['body'] = $transformed;
665 q("UPDATE `item` SET `body` = '%s' WHERE `plink` = '%s' AND `uid` = %d AND `contact-id` = %d",
666 dbesc($item['body']),
667 dbesc($item['plink']),
668 intval($item['uid']),
669 intval($item['contact-id']));
672 function retriever_content($a) {
674 $a->page['content'] .= "<p>Please log in</p>";
677 if ($a->argv[1] === 'help') {
678 $feeds = q("SELECT `id`, `name`, `thumb` FROM contact WHERE `uid` = %d AND `network` = 'feed'",
680 foreach ($feeds as $k=>$v) {
681 $feeds[$k]['url'] = $a->get_baseurl() . '/retriever/' . $v['id'];
683 $template = get_markup_template('/help.tpl', 'addon/retriever/');
684 $a->page['content'] .= replace_macros($template, array(
685 '$config' => $a->get_baseurl() . '/settings/addon',
686 '$feeds' => $feeds));
690 $retriever = get_retriever($a->argv[1], local_user(), false);
692 if (x($_POST["id"])) {
693 $retriever = get_retriever($a->argv[1], local_user(), true);
694 $retriever["data"] = array();
695 foreach (array('pattern', 'replace', 'enable', 'images', 'customxslt') as $setting) {
696 if (x($_POST['retriever_' . $setting])) {
697 $retriever["data"][$setting] = $_POST['retriever_' . $setting];
700 foreach ($_POST as $k=>$v) {
701 if (preg_match("/retriever-(include|exclude)-(\d+)-(element|attribute|value)/", $k, $matches)) {
702 $retriever['data'][$matches[1]][intval($matches[2])][$matches[3]] = $v;
705 // You've gotta have an element, even if it's just "*"
706 foreach ($retriever['data']['include'] as $k=>$clause) {
707 if (!$clause['element']) {
708 unset($retriever['data']['include'][$k]);
711 foreach ($retriever['data']['exclude'] as $k=>$clause) {
712 if (!$clause['element']) {
713 unset($retriever['data']['exclude'][$k]);
716 q("UPDATE `retriever_rule` SET `data`='%s' WHERE `id` = %d",
717 dbesc(json_encode($retriever["data"])), intval($retriever["id"]));
718 $a->page['content'] .= "<p><b>Settings Updated";
719 if (x($_POST["retriever_retrospective"])) {
720 apply_retrospective($retriever, $_POST["retriever_retrospective"]);
721 $a->page['content'] .= " and retrospectively applied to " . $_POST["apply"] . " posts";
723 $a->page['content'] .= ".</p></b>";
726 $template = get_markup_template('/rule-config.tpl', 'addon/retriever/');
727 $a->page['content'] .= replace_macros($template, array(
731 $retriever['data']['enable']),
735 $retriever["data"]['pattern'],
736 t('Regular expression matching part of the URL to replace')),
740 $retriever["data"]['replace'],
741 t('Text to replace matching part of above regular expression')),
744 t('Download Images'),
745 $retriever['data']['images']),
746 '$retrospective' => array(
747 'retriever_retrospective',
748 t('Retrospectively Apply'),
750 t('Reapply the rules to this number of posts')),
751 '$customxslt' => array(
752 'retriever_customxslt',
754 $retriever['data']['customxslt'],
755 t("When standard rules aren't enough, apply custom XSLT to the article")),
756 '$title' => t('Retrieve Feed Content'),
757 '$help' => $a->get_baseurl() . '/retriever/help',
758 '$help_t' => t('Get Help'),
759 '$submit_t' => t('Submit'),
760 '$submit' => t('Save Settings'),
761 '$id' => ($retriever["id"] ? $retriever["id"] : "create"),
762 '$tag_t' => t('Tag'),
763 '$attribute_t' => t('Attribute'),
764 '$value_t' => t('Value'),
765 '$add_t' => t('Add'),
766 '$remove_t' => t('Remove'),
767 '$include_t' => t('Include'),
768 '$include' => $retriever['data']['include'],
769 '$exclude_t' => t('Exclude'),
770 '$exclude' => $retriever["data"]['exclude']));
775 function retriever_contact_photo_menu($a, &$args) {
779 if ($args["contact"]["network"] == "feed") {
780 $args["menu"][ 'retriever' ] = array(t('Retriever'), $a->get_baseurl() . '/retriever/' . $args["contact"]['id']);
784 function retriever_post_remote_hook(&$a, &$item) {
785 logger('retriever_post_remote_hook: ' . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id'], LOGGER_DEBUG);
787 $retriever = get_retriever($item['contact-id'], $item["uid"], false);
789 retriever_on_item_insert($retriever, $item);
792 if (get_pconfig($item["uid"], 'retriever', 'oembed')) {
793 // Convert to HTML and back to take advantage of bbcode's resolution of oembeds.
794 $body = html2bbcode(bbcode($item['body']));
796 $item['body'] = $body;
799 if (get_pconfig($item["uid"], 'retriever', 'all_photos')) {
800 retrieve_images($item, null);
803 retriever_check_item_completed($item);
806 function retriever_plugin_settings(&$a,&$s) {
807 $all_photos = get_pconfig(local_user(), 'retriever', 'all_photos');
808 $oembed = get_pconfig(local_user(), 'retriever', 'oembed');
809 $template = get_markup_template('/settings.tpl', 'addon/retriever/');
810 $s .= replace_macros($template, array(
811 '$allphotos' => array(
812 'retriever_all_photos',
815 t('Check this to retrieve photos for all posts')),
820 t('Check this to attempt to retrieve embedded content for all posts - useful e.g. for Facebook posts')),
821 '$submit' => t('Save Settings'),
822 '$title' => t('Retriever Settings'),
823 '$help' => $a->get_baseurl() . '/retriever/help'));
826 function retriever_plugin_settings_post($a,$post) {
827 if ($_POST['retriever_all_photos']) {
828 set_pconfig(local_user(), 'retriever', 'all_photos', $_POST['retriever_all_photos']);
831 del_pconfig(local_user(), 'retriever', 'all_photos');
833 if ($_POST['retriever_oembed']) {
834 set_pconfig(local_user(), 'retriever', 'oembed', $_POST['retriever_oembed']);
837 del_pconfig(local_user(), 'retriever', 'oembed');