3 * Name: Retrieve Feed Content
4 * Description: Follow the permalink of RSS/Atom feed items and replace the summary with the full content.
6 * Author: Matthew Exon <http://mat.exon.name>
9 function retriever_install() {
10 register_hook('plugin_settings', 'addon/retriever/retriever.php', 'retriever_plugin_settings');
11 register_hook('plugin_settings_post', 'addon/retriever/retriever.php', 'retriever_plugin_settings_post');
12 register_hook('post_remote', 'addon/retriever/retriever.php', 'retriever_post_remote_hook');
13 register_hook('contact_photo_menu', 'addon/retriever/retriever.php', 'retriever_contact_photo_menu');
14 register_hook('cron', 'addon/retriever/retriever.php', 'retriever_cron');
16 $schema = file_get_contents(dirname(__file__).'/database.sql');
17 $arr = explode(';', $schema);
18 foreach ($arr as $a) {
22 $r = q("SELECT `id` FROM `pconfig` WHERE `cat` LIKE 'retriever_%%'");
23 if (count($r) || (get_config('retriever', 'dbversion') == '0.1')) {
24 $retrievers = array();
25 $r = q("SELECT SUBSTRING(`cat`, 10) AS `contact`, `k`, `v` FROM `pconfig` WHERE `cat` LIKE 'retriever%%'");
27 $retrievers[$rr['contact']][$rr['k']] = $rr['v'];
29 foreach ($retrievers as $k => $v) {
30 $rr = q("SELECT `uid` FROM `contact` WHERE `id` = %d", intval($k));
33 q("INSERT INTO `retriever_rule` (`uid`, `contact-id`, `data`) VALUES (%d, %d, '%s')",
34 intval($uid), intval($k), dbesc(json_encode($v)));
36 q("DELETE FROM `pconfig` WHERE `cat` LIKE 'retriever%%'");
38 if (get_config('retriever', 'dbversion') == '0.2') {
39 q("ALTER TABLE `retriever_resource` DROP COLUMN `retriever`");
41 if (get_config('retriever', 'dbversion') == '0.3') {
42 q("ALTER TABLE `retriever_item` MODIFY COLUMN `item-uri` varchar(800) CHARACTER SET ascii NOT NULL");
43 q("ALTER TABLE `retriever_resource` MODIFY COLUMN `url` varchar(800) CHARACTER SET ascii NOT NULL");
45 if (get_config('retriever', 'dbversion') == '0.4') {
46 q("ALTER TABLE `retriever_item` ADD COLUMN `finished` tinyint(1) unsigned NOT NULL DEFAULT '0'");
48 if (get_config('retriever', 'dbversion') == '0.5') {
49 q('ALTER TABLE `retriever_resource` CHANGE `created` `created` timestamp NOT NULL DEFAULT now()');
50 q('ALTER TABLE `retriever_resource` CHANGE `completed` `completed` timestamp NULL DEFAULT NULL');
51 q('ALTER TABLE `retriever_resource` CHANGE `last-try` `last-try` timestamp NULL DEFAULT NULL');
52 q('ALTER TABLE `retriever_item` DROP KEY `all`');
53 q('ALTER TABLE `retriever_item` ADD KEY `all` (`item-uri`, `item-uid`, `contact-id`)');
55 if (get_config('retriever', 'dbversion') == '0.6') {
56 q('ALTER TABLE `retriever_item` CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin');
57 q('ALTER TABLE `retriever_item` CHANGE `item-uri` `item-uri` varchar(800) CHARACTER SET ascii COLLATE ascii_bin NOT NULL');
58 q('ALTER TABLE `retriever_resource` CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin');
59 q('ALTER TABLE `retriever_resource` CHANGE `url` `url` varchar(800) CHARACTER SET ascii COLLATE ascii_bin NOT NULL');
60 q('ALTER TABLE `retriever_rule` CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin');
62 if (get_config('retriever', 'dbversion') == '0.7') {
63 $r = q("SELECT `id`, `data` FROM `retriever_rule`");
65 logger('retriever_install: retriever ' . $rr['id'] . ' old config ' . $rr['data'], LOGGER_DATA);
66 $data = json_decode($rr['data'], true);
67 if ($data['pattern']) {
69 if (preg_match("/\/(.*)\//", $data['pattern'], $matches)) {
70 $data['pattern'] = $matches[1];
75 foreach (explode('|', $data['match']) as $component) {
77 if (preg_match("/([A-Za-z][A-Za-z0-9]*)\[@([A-Za-z][a-z0-9]*)='([^']*)'\]/", $component, $matches)) {
79 'element' => $matches[1],
80 'attribute' => $matches[2],
81 'value' => $matches[3]);
83 if (preg_match("/([A-Za-z][A-Za-z0-9]*)\[contains(concat(' ',normalize-space(@class),' '),' ([^ ']+) ')]/", $component, $matches)) {
85 'element' => $matches[1],
86 'attribute' => $matches[2],
87 'value' => $matches[3]);
90 $data['include'] = $include;
91 unset($data['match']);
93 if ($data['remove']) {
95 foreach (explode('|', $data['remove']) as $component) {
97 if (preg_match("/([A-Za-z][A-Za-z0-9]*)\[@([A-Za-z][a-z0-9]*)='([^']*)'\]/", $component, $matches)) {
99 'element' => $matches[1],
100 'attribute' => $matches[2],
101 'value' => $matches[3]);
103 if (preg_match("/([A-Za-z][A-Za-z0-9]*)\[contains(concat(' ',normalize-space(@class),' '),' ([^ ']+) ')]/", $component, $matches)) {
105 'element' => $matches[1],
106 'attribute' => $matches[2],
107 'value' => $matches[3]);
110 $data['exclude'] = $exclude;
111 unset($data['remove']);
113 $r = q('UPDATE `retriever_rule` SET `data` = "%s" WHERE `id` = %d', dbesc(json_encode($data)), $rr['id']);
114 logger('retriever_install: retriever ' . $rr['id'] . ' new config ' . json_encode($data), LOGGER_DATA);
117 set_config('retriever', 'dbversion', '0.8');
120 function retriever_uninstall() {
121 unregister_hook('plugin_settings', 'addon/retriever/retriever.php', 'retriever_plugin_settings');
122 unregister_hook('plugin_settings_post', 'addon/retriever/retriever.php', 'retriever_plugin_settings_post');
123 unregister_hook('post_remote', 'addon/retriever/retriever.php', 'retriever_post_remote_hook');
124 unregister_hook('plugin_settings', 'addon/retriever/retriever.php', 'retriever_plugin_settings');
125 unregister_hook('plugin_settings_post', 'addon/retriever/retriever.php', 'retriever_plugin_settings_post');
126 unregister_hook('contact_photo_menu', 'addon/retriever/retriever.php', 'retriever_contact_photo_menu');
127 unregister_hook('cron', 'addon/retriever/retriever.php', 'retriever_cron');
130 function retriever_module() {}
132 function retriever_cron($a, $b) {
133 // 100 is a nice sane number. Maybe this should be configurable.
134 retriever_retrieve_items(100);
138 $retriever_item_count = 0;
140 function retriever_retrieve_items($max_items) {
141 global $retriever_item_count;
143 $retriever_schedule = array(array(1,'minute'),
151 $schedule_clauses = array();
152 for ($i = 0; $i < count($retriever_schedule); $i++) {
153 $num = $retriever_schedule[$i][0];
154 $unit = $retriever_schedule[$i][1];
155 array_push($schedule_clauses,
156 '(`num-tries` = ' . $i . ' AND TIMESTAMPADD(' . dbesc($unit) .
157 ', ' . intval($num) . ', `last-try`) < now())');
160 $retrieve_items = $max_items - $retriever_item_count;
162 $r = q("SELECT * FROM `retriever_resource` WHERE `completed` IS NULL AND (`last-try` IS NULL OR %s) ORDER BY `last-try` ASC LIMIT %d",
163 dbesc(implode($schedule_clauses, ' OR ')),
164 intval($retrieve_items));
165 if (count($r) == 0) {
168 foreach ($r as $rr) {
169 retrieve_resource($rr);
170 $retriever_item_count++;
172 $retrieve_items = $max_items - $retriever_item_count;
174 while ($retrieve_items > 0);
176 /* Look for items that are waiting even though the resource has
177 * completed. This usually happens because we've been asked to
178 * retrospectively apply a config change. It could also happen
179 * due to a cron job dying or something. */
180 $r = q("SELECT retriever_resource.`id` as resource, retriever_item.`id` as item FROM retriever_resource, retriever_item, retriever_rule WHERE retriever_item.`finished` = 0 AND retriever_item.`resource` = retriever_resource.`id` AND retriever_resource.`completed` IS NOT NULL AND retriever_item.`contact-id` = retriever_rule.`contact-id` AND retriever_item.`item-uid` = retriever_rule.`uid` LIMIT %d",
181 intval($retrieve_items));
185 foreach ($r as $rr) {
186 $resource = q("SELECT * FROM retriever_resource WHERE `id` = %d", $rr['resource']);
187 $retriever_item = retriever_get_retriever_item($rr['item']);
188 if (!$retriever_item) {
189 logger('retriever_retrieve_items: no retriever item with id ' . $rr['item']);
192 $item = retriever_get_item($retriever_item);
194 logger('retriever_retrieve_items: no item ' . $retriever_item['item-uri']);
197 $retriever = get_retriever($item['contact-id'], $item['uid']);
199 logger('retriever_retrieve_items: no retriever for item ' .
200 $retriever_item['item-uri'] . ' ' . $retriever_item['uid'] . ' ' . $item['contact-id']);
203 retriever_apply_completed_resource_to_item($retriever, $item, $resource[0]);
204 q("UPDATE `retriever_item` SET `finished` = 1 WHERE id = %d",
205 intval($retriever_item['id']));
206 retriever_check_item_completed($item);
210 function retriever_tidy() {
211 q("DELETE FROM retriever_resource WHERE completed IS NOT NULL AND completed < DATE_SUB(now(), INTERVAL 1 WEEK)");
212 q("DELETE FROM retriever_resource WHERE completed IS NULL AND created < DATE_SUB(now(), INTERVAL 3 MONTH)");
214 $r = q("SELECT retriever_item.id FROM retriever_item LEFT OUTER JOIN retriever_resource ON (retriever_item.resource = retriever_resource.id) WHERE retriever_resource.id is null");
215 foreach ($r as $rr) {
216 q('DELETE FROM retriever_item WHERE id = %d', intval($rr['id']));
220 function retrieve_resource($resource) {
221 logger('retrieve_resource: ' . ($resource['num-tries'] + 1) .
222 ' attempt at resource ' . $resource['id'] . ' ' . $resource['url'], LOGGER_DEBUG);
223 q("UPDATE `retriever_resource` SET `last-try` = now(), `num-tries` = `num-tries` + 1 WHERE id = %d",
224 intval($resource['id']));
225 $data = fetch_url($resource['url'], $resource['binary'], $resource['type']);
226 $resource['type'] = get_app()->get_curl_content_type();
228 $resource['data'] = $data;
229 q("UPDATE `retriever_resource` SET `completed` = now(), `data` = '%s', `type` = '%s' WHERE id = %d",
230 dbesc($data), dbesc($resource['type']), intval($resource['id']));
231 retriever_resource_completed($resource);
235 function get_retriever($contact_id, $uid, $create = false) {
236 $r = q("SELECT * FROM `retriever_rule` WHERE `contact-id` = %d AND `uid` = %d",
237 intval($contact_id), intval($uid));
239 $r[0]['data'] = json_decode($r[0]['data'], true);
243 q("INSERT INTO `retriever_rule` (`uid`, `contact-id`) VALUES (%d, %d)",
244 intval($uid), intval($contact_id));
245 $r = q("SELECT * FROM `retriever_rule` WHERE `contact-id` = %d AND `uid` = %d",
246 intval($contact_id), intval($uid));
251 function retriever_get_retriever_item($id) {
252 $retriever_items = q("SELECT * FROM `retriever_item` WHERE id = %d", intval($id));
253 if (count($retriever_items) != 1) {
254 logger('retriever_get_retriever_item: unable to find retriever_item ' . $id, LOGGER_NORMAL);
257 return $retriever_items[0];
260 function retriever_get_item($retriever_item) {
261 $items = q("SELECT * FROM `item` WHERE `uri` = '%s' AND `uid` = %d AND `contact-id` = %d",
262 dbesc($retriever_item['item-uri']),
263 intval($retriever_item['item-uid']),
264 intval($retriever_item['contact-id']));
265 if (count($items) != 1) {
266 logger('retriever_get_item: unexpected number of results ' .
267 count($items) . " when searching for item $uri $uid $cid", LOGGER_NORMAL);
273 function retriever_item_completed($retriever_item_id, $resource) {
274 logger('retriever_item_completed: id ' . $retriever_item_id . ' url ' . $resource['url'], LOGGER_DEBUG);
276 $retriever_item = retriever_get_retriever_item($retriever_item_id);
277 if (!$retriever_item) {
280 // Note: the retriever might be null. Doesn't matter.
281 $retriever = get_retriever($retriever_item['contact-id'], $retriever_item['item-uid']);
282 $item = retriever_get_item($retriever_item);
287 retriever_apply_completed_resource_to_item($retriever, $item, $resource);
289 q("UPDATE `retriever_item` SET `finished` = 1 WHERE id = %d",
290 intval($retriever_item['id']));
291 retriever_check_item_completed($item);
294 function retriever_resource_completed($resource) {
295 logger('retriever_resource_completed: id ' . $resource['id'] . ' url ' . $resource['url'], LOGGER_DEBUG);
296 $r = q("SELECT `id` FROM `retriever_item` WHERE `resource` = %d", $resource['id']);
297 foreach ($r as $rr) {
298 retriever_item_completed($rr['id'], $resource);
302 function apply_retrospective($retriever, $num) {
303 $r = q("SELECT * FROM `item` WHERE `contact-id` = %d ORDER BY `received` DESC LIMIT %d",
304 intval($retriever['contact-id']), intval($num));
305 foreach ($r as $item) {
306 q('UPDATE `item` SET `visible` = 0 WHERE `id` = %d', $item['id']);
307 retriever_on_item_insert($retriever, $item);
311 function retriever_on_item_insert($retriever, &$item) {
312 if (!$retriever || !$retriever['id']) {
313 logger('retriever_on_item_insert: No retriever supplied', LOGGER_NORMAL);
316 if (!$retriever["data"]['enable'] == "on") {
319 if ($retriever["data"]['pattern']) {
320 $url = preg_replace('/' . $retriever["data"]['pattern'] . '/', $retriever["data"]['replace'], $item['plink']);
321 logger('retriever_on_item_insert: Changed ' . $item['plink'] . ' to ' . $url, LOGGER_DATA);
324 $url = $item['plink'];
327 $resource = add_retriever_resource($url);
328 $retriever_item_id = add_retriever_item($item, $resource);
331 function add_retriever_resource($url, $binary = false) {
332 logger('add_retriever_resource: ' . $url, LOGGER_DEBUG);
333 $r = q("SELECT * FROM `retriever_resource` WHERE `url` = '%s'", dbesc($url));
336 logger('add_retriever_resource: Resource ' . $url . ' already requested', LOGGER_DEBUG);
340 q("INSERT INTO `retriever_resource` (`binary`, `url`) " .
341 "VALUES (%d, '%s')", intval($binary ? 1 : 0), dbesc($url));
342 $r = q("SELECT * FROM `retriever_resource` WHERE `url` = '%s'", dbesc($url));
347 function add_retriever_item(&$item, $resource) {
348 logger('add_retriever_item: ' . $resource['url'] . ' for ' . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id'], LOGGER_DEBUG);
350 q("INSERT INTO `retriever_item` (`item-uri`, `item-uid`, `contact-id`, `resource`) " .
351 "VALUES ('%s', %d, %d, %d)",
352 dbesc($item['uri']), intval($item['uid']), intval($item['contact-id']), intval($resource["id"]));
353 $r = q("SELECT id FROM `retriever_item` WHERE " .
354 "`item-uri` = '%s' AND `item-uid` = %d AND `contact-id` = %d AND `resource` = %d ORDER BY id DESC",
355 dbesc($item['uri']), intval($item['uid']), intval($item['contact-id']), intval($resource['id']));
357 logger("add_retriever_item: couldn't create retriever item for " .
358 $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id'],
362 logger('add_retriever_item: created retriever_item ' . $r[0]['id'] . ' for item ' . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id'], LOGGER_DEBUG);
366 function retriever_get_encoding($resource) {
368 if (preg_match('/charset=(.*)/', $resource['type'], $matches)) {
369 return trim(array_pop($matches));
374 function retriever_construct_xpath($spec) {
375 if (gettype($spec) != "array") {
378 $components = array();
379 foreach ($spec as $clause) {
380 if (!$clause['attribute']) {
381 $components[] = $clause['element'];
384 if ($clause['attribute'] === 'class') {
387 "[contains(concat(' ', normalize-space(@class), ' '), ' " .
388 $clause['value'] . " ')]";
392 $clause['element'] . '[@' .
393 $clause['attribute'] . "='" .
394 $clause['value'] . "']";
397 // It would be better to do this in smarty3 in extract.tpl
398 return implode('|', $components);
401 function retriever_apply_dom_filter($retriever, &$item, $resource) {
402 logger('retriever_apply_dom_filter: applying XSLT to ' . $item['id'] . ' ' . $item['plink'], LOGGER_DEBUG);
403 require_once('include/html2bbcode.php');
405 if (!$retriever['data']['include']) {
408 if (!$resource['data']) {
409 logger('retriever_apply_dom_filter: no text to work with', LOGGER_NORMAL);
413 $encoding = retriever_get_encoding($resource);
414 logger('@@@ item type ' . $resource['type'] . ' encoding ' . $encoding);
415 $extracter_template = get_markup_template('extract.tpl', 'addon/retriever/');
416 $doc = new DOMDocument('1.0', 'utf-8');
417 if (strpos($resource['type'], 'html') !== false) {
418 @$doc->loadHTML($resource['data']);
421 $doc->loadXML($resource['data']);
423 logger('@@@ actual encoding of document is ' . $doc->encoding);
425 $components = parse_url($item['plink']);
426 $rooturl = $components['scheme'] . "://" . $components['host'];
427 $dirurl = $rooturl . dirname($components['path']) . "/";
429 $params = array('$include' => retriever_construct_xpath($retriever['data']['include']),
430 '$exclude' => retriever_construct_xpath($retriever['data']['exclude']),
431 '$pageurl' => $item['plink'],
432 '$dirurl' => $dirurl,
433 '$rooturl' => $rooturl);
434 $xslt = replace_macros($extracter_template, $params);
435 $xmldoc = new DOMDocument();
436 $xmldoc->loadXML($xslt);
437 $xp = new XsltProcessor();
438 $xp->importStylesheet($xmldoc);
439 $transformed = $xp->transformToXML($doc);
440 $item['body'] = html2bbcode($transformed);
441 if (!strlen($item['body'])) {
442 logger('retriever_apply_dom_filter retriever ' . $retriever['id'] . ' item ' . $item['id'] . ': output was empty', LOGGER_NORMAL);
445 $item['body'] .= "\n\n" . t('Retrieved') . ' ' . date("Y-m-d") . ': [url=';
446 $item['body'] .= $item['plink'];
447 $item['body'] .= ']' . $item['plink'] . '[/url]';
448 q("UPDATE `item` SET `body` = '%s' WHERE `id` = %d",
449 dbesc($item['body']), intval($item['id']));
452 function retrieve_images(&$item) {
454 preg_match_all("/\[img\=([0-9]*)x([0-9]*)\](.*?)\[\/img\]/ism", $item["body"], $matches1);
456 preg_match_all("/\[img\](.*?)\[\/img\]/ism", $item["body"], $matches2);
457 $matches = array_merge($matches1[3], $matches2[1]);
458 logger('retrieve_images: found ' . count($matches) . ' images for item ' . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id'], LOGGER_DEBUG);
459 foreach ($matches as $url) {
460 if (strpos($url, get_app()->get_baseurl()) === FALSE) {
461 $resource = add_retriever_resource($url, true);
462 if (!$resource['completed']) {
463 add_retriever_item($item, $resource);
466 retriever_transform_images($item, $resource);
472 function retriever_check_item_completed(&$item)
474 $r = q('SELECT count(*) FROM retriever_item WHERE `item-uri` = "%s" ' .
475 'AND `item-uid` = %d AND `contact-id` = %d AND `finished` = 0',
476 dbesc($item['uri']), intval($item['uid']),
477 intval($item['contact-id']));
478 $waiting = $r[0]['count(*)'];
479 logger('retriever_check_item_completed: item ' . $item['uri'] . ' ' . $item['uid']
480 . ' '. $item['contact-id'] . ' waiting for ' . $waiting . ' resources', LOGGER_DEBUG);
481 $old_visible = $item['visible'];
482 $item['visible'] = $waiting ? 0 : 1;
483 if (($item['id'] > 0) && ($old_visible != $item['visible'])) {
484 logger('retriever_check_item_completed: changing visible flag to ' . $item['visible'] . ' and invoking notifier ("edit_post", ' . $item['id'] . ')', LOGGER_DEBUG);
485 q("UPDATE `item` SET `visible` = %d WHERE `id` = %d",
486 intval($item['visible']),
487 intval($item['id']));
488 // disable due to possible security issue
489 // proc_run('php', "include/notifier.php", 'edit_post', $item['id']);
493 function retriever_apply_completed_resource_to_item($retriever, &$item, $resource) {
494 logger('retriever_apply_completed_resource_to_item: retriever ' .
495 ($retriever ? $retriever['id'] : 'none') .
496 ' resource ' . $resource['url'] . ' plink ' . $item['plink'], LOGGER_DEBUG);
497 if (strpos($resource['type'], 'image') !== false) {
498 retriever_transform_images($item, $resource);
503 if ((strpos($resource['type'], 'html') !== false) ||
504 (strpos($resource['type'], 'xml') !== false)) {
505 retriever_apply_dom_filter($retriever, $item, $resource);
506 if ($retriever["data"]['images'] ) {
507 retrieve_images($item);
512 function retriever_store_photo($item, &$resource) {
513 $hash = photo_new_resource();
515 if (class_exists('Imagick')) {
517 $image = new Imagick();
518 $image->readImageBlob($resource['data']);
519 $resource['width'] = $image->getImageWidth();
520 $resource['height'] = $image->getImageHeight();
522 catch (Exception $e) {
523 logger("ImageMagick couldn't process image " . $resource['id'] . " " . $resource['url'] . ' length ' . strlen($resource['data']) . ': ' . $e->getMessage(), LOGGER_DEBUG);
527 if (!array_key_exists('width', $resource)) {
528 $image = @imagecreatefromstring($resource['data']);
529 if ($image === false) {
530 logger("Couldn't process image " . $resource['id'] . " " . $resource['url'], LOGGER_DEBUG);
533 $resource['width'] = imagesx($image);
534 $resource['height'] = imagesy($image);
535 imagedestroy($image);
538 $url_components = parse_url($resource['url']);
539 $filename = basename($url_components['path']);
540 if (!strlen($filename)) {
543 $r = q("INSERT INTO `photo`
544 ( `uid`, `contact-id`, `guid`, `resource-id`, `created`, `edited`, `filename`, `type`, `album`, `height`, `width`, `datasize`, `data` )
545 VALUES ( %d, %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, %d, %d, '%s' )",
546 intval($item['item-uid']),
547 intval($item['contact-id']),
550 dbesc(datetime_convert()),
551 dbesc(datetime_convert()),
553 dbesc($resource['type']),
554 dbesc('Retrieved Images'),
555 intval($resource['height']),
556 intval($resource['width']),
557 intval(strlen($resource['data'])),
558 dbesc($resource['data'])
564 function retriever_transform_images(&$item, $resource) {
565 require_once('include/Photo.php');
567 if (!$resource["data"]) {
568 logger('retriever_transform_images: no data available for '
569 . $resource['id'] . ' ' . $resource['url'], LOGGER_NORMAL);
573 $hash = retriever_store_photo($item, $resource);
574 if ($hash === false) {
575 logger('retriever_transform_images: unable to store photo '
576 . $resource['id'] . ' ' . $resource['url'], LOGGER_NORMAL);
580 $new_url = get_app()->get_baseurl() . '/photo/' . $hash;
581 logger('retriever_transform_images: replacing ' . $resource['url'] . ' with ' .
582 $new_url . ' in item ' . $item['plink'], LOGGER_DEBUG);
583 $transformed = str_replace($resource["url"], $new_url, $item['body']);
584 if ($transformed === $item['body']) {
588 $item['body'] = $transformed;
589 q("UPDATE `item` SET `body` = '%s' WHERE `plink` = '%s' AND `uid` = %d AND `contact-id` = %d",
590 dbesc($item['body']),
591 dbesc($item['plink']),
592 intval($item['uid']),
593 intval($item['contact-id']));
596 function retriever_content($a) {
598 $a->page['content'] .= "<p>Please log in</p>";
601 if ($a->argv[1] === 'help') {
602 $feeds = q("SELECT `id`, `name`, `thumb` FROM contact WHERE `uid` = %d AND `network` = 'feed'",
604 foreach ($feeds as $k=>$v) {
605 $feeds[$k]['url'] = $a->get_baseurl() . '/retriever/' . $v['id'];
607 $template = get_markup_template('/help.tpl', 'addon/retriever/');
608 $a->page['content'] .= replace_macros($template, array(
609 '$config' => $a->get_baseurl() . '/settings/addon',
610 '$feeds' => $feeds));
614 $retriever = get_retriever($a->argv[1], local_user(), false);
616 if (x($_POST["id"])) {
617 $retriever = get_retriever($a->argv[1], local_user(), true);
618 $retriever["data"] = array();
619 foreach (array('pattern', 'replace', 'enable', 'images') as $setting) {
620 if (x($_POST['retriever_' . $setting])) {
621 $retriever["data"][$setting] = $_POST['retriever_' . $setting];
624 foreach ($_POST as $k=>$v) {
625 if (preg_match("/retriever-(include|exclude)-(\d+)-(element|attribute|value)/", $k, $matches)) {
626 $retriever['data'][$matches[1]][intval($matches[2])][$matches[3]] = $v;
629 // You've gotta have an element, even if it's just "*"
630 foreach ($retriever['data']['include'] as $k=>$clause) {
631 if (!$clause['element']) {
632 unset($retriever['data']['include'][$k]);
635 foreach ($retriever['data']['exclude'] as $k=>$clause) {
636 if (!$clause['element']) {
637 unset($retriever['data']['exclude'][$k]);
640 q("UPDATE `retriever_rule` SET `data`='%s' WHERE `id` = %d",
641 dbesc(json_encode($retriever["data"])), intval($retriever["id"]));
642 $a->page['content'] .= "<p><b>Settings Updated";
643 if (x($_POST["retriever_retrospective"])) {
644 apply_retrospective($retriever, $_POST["retriever_retrospective"]);
645 $a->page['content'] .= " and retrospectively applied to " . $_POST["apply"] . " posts";
647 $a->page['content'] .= ".</p></b>";
650 $template = get_markup_template('/rule-config.tpl', 'addon/retriever/');
651 $a->page['content'] .= replace_macros($template, array(
655 $retriever['data']['enable']),
659 $retriever["data"]['pattern'],
660 t('Regular expression matching part of the URL to replace')),
664 $retriever["data"]['replace'],
665 t('Text to replace matching part of above regular expression')),
668 t('Download Images'),
669 $retriever['data']['images']),
670 '$retrospective' => array(
671 'retriever_retrospective',
672 t('Retrospectively Apply'),
674 t('Reapply the rules to this number of posts')),
675 '$title' => t('Retrieve Feed Content'),
676 '$help' => $a->get_baseurl() . '/retriever/help',
677 '$submit' => t('Submit'),
678 '$id' => ($retriever["id"] ? $retriever["id"] : "create"),
679 '$tag_t' => t('Tag'),
680 '$attribute_t' => t('Attribute'),
681 '$value_t' => t('Value'),
682 '$add_t' => t('Add'),
683 '$remove_t' => t('Remove'),
684 '$include_t' => t('Include'),
685 '$include' => $retriever['data']['include'],
686 '$exclude_t' => t('Exclude'),
687 '$exclude' => $retriever["data"]['exclude']));
692 function retriever_contact_photo_menu($a, &$args) {
696 if ($args["contact"]["network"] == "feed") {
697 $args["menu"][ 'retriever' ] = array(t('Retriever'), $a->get_baseurl() . '/retriever/' . $args["contact"]['id']);
701 function retriever_post_remote_hook(&$a, &$item) {
702 logger('retriever_post_remote_hook: ' . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id'], LOGGER_DEBUG);
704 $retriever = get_retriever($item['contact-id'], $item["uid"], false);
706 retriever_on_item_insert($retriever, $item);
709 if (get_pconfig($item["uid"], 'retriever', 'all_photos')) {
710 retrieve_images($item, null);
713 retriever_check_item_completed($item);
716 function retriever_plugin_settings(&$a,&$s) {
717 $all_photos = get_pconfig(local_user(), 'retriever', 'all_photos');
718 $all_photos_mu = ($all_photos == 'on') ? ' checked="true"' : '';
719 $template = get_markup_template('/settings.tpl', 'addon/retriever/');
720 $s .= replace_macros($template, array(
721 '$submit' => t('Submit'),
722 '$title' => t('Retriever Settings'),
723 '$help' => $a->get_baseurl() . '/retriever/help',
724 '$all_photos' => $all_photos_mu,
725 '$all_photos_t' => t('All Photos')));
728 function retriever_plugin_settings_post($a,$post) {
729 if ($_POST['all_photos']) {
730 set_pconfig(local_user(), 'retriever', 'all_photos', $_POST['all_photos']);
733 del_pconfig(local_user(), 'retriever', 'all_photos');