Merge pull request #2253 from annando/1601-network-performance

Reducing network request, increasing page performance

diff --git a/boot.php b/boot.php
index 7d8c7040de9a1cc67df908588177dc54870b55cb..2242ba476d85f06e811eb872b4f435f5c2625ec0 100644 (file)
--- a/boot.php
+++ b/boot.php
@@ -36,7 +36,7 @@ define ( 'FRIENDICA_PLATFORM',     'Friendica');
 define ( 'FRIENDICA_CODENAME',     'Asparagus');
 define ( 'FRIENDICA_VERSION',      '3.5-dev' );
 define ( 'DFRN_PROTOCOL_VERSION',  '2.23'    );
-define ( 'DB_UPDATE_VERSION',      1192      );
+define ( 'DB_UPDATE_VERSION',      1193      );
 
 /**
  * @brief Constant with a HTML line break.
@@ -467,6 +467,7 @@ class App {
        public  $is_tablet;
        public  $is_friendica_app;
        public  $performance = array();
+       public  $callstack = array();
 
        public $nav_sel;
 
@@ -554,6 +555,12 @@ class App {
                $this->performance["marktime"] = 0;
                $this->performance["markstart"] = microtime(true);
 
+               $this->callstack["database"] = array();
+               $this->callstack["network"] = array();
+               $this->callstack["file"] = array();
+               $this->callstack["rendering"] = array();
+               $this->callstack["parser"] = array();
+
                $this->config = array();
                $this->page = array();
                $this->pager= array();
@@ -903,6 +910,10 @@ class App {
        }
 
        function get_cached_avatar_image($avatar_image){
+               return $avatar_image;
+
+               // The following code is deactivated. It doesn't seem to make any sense and it slows down the system.
+               /*
                if($this->cached_profile_image[$avatar_image])
                        return $this->cached_profile_image[$avatar_image];
 
@@ -922,6 +933,7 @@ class App {
                        }
                }
                return $this->cached_profile_image[$avatar_image];
+               */
        }
 
 
@@ -1016,6 +1028,20 @@ class App {
 
                $this->performance[$value] += (float)$duration;
                $this->performance["marktime"] += (float)$duration;
+
+               // Trace the different functions with their timestamps
+               $trace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 5);
+
+               array_shift($trace);
+
+               $function = array();
+               foreach ($trace AS $func)
+                       $function[] = $func["function"];
+
+               $function = implode(", ", $function);
+
+               $this->callstack[$value][$function] += (float)$duration;
+
        }
 
        function mark_timestamp($mark) {

diff --git a/database.sql b/database.sql
index df5ec74e29d159ddfeab83eeae06e18e1efc0309..70b315ea244f0be305b8647be65e3e1f4fee2c73 100644 (file)
--- a/database.sql
+++ b/database.sql
@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 3.5-dev (Asparagus)
--- DB_UPDATE_VERSION 1192
+-- DB_UPDATE_VERSION 1193
 -- ------------------------------------------
 
 
@@ -685,6 +685,30 @@ CREATE TABLE IF NOT EXISTS `notify-threads` (
         INDEX `receiver-uid` (`receiver-uid`)
 ) DEFAULT CHARSET=utf8;
 
+--
+-- TABLE oembed
+--
+CREATE TABLE IF NOT EXISTS `oembed` (
+       `url` varchar(255) NOT NULL,
+       `content` text NOT NULL,
+       `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
+        PRIMARY KEY(`url`),
+        INDEX `created` (`created`)
+) DEFAULT CHARSET=utf8;
+
+--
+-- TABLE parsed_url
+--
+CREATE TABLE IF NOT EXISTS `parsed_url` (
+       `url` varchar(255) NOT NULL,
+       `guessing` tinyint(1) NOT NULL DEFAULT 0,
+       `oembed` tinyint(1) NOT NULL DEFAULT 0,
+       `content` text NOT NULL,
+       `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
+        PRIMARY KEY(`url`,`guessing`,`oembed`),
+        INDEX `created` (`created`)
+) DEFAULT CHARSET=utf8;
+
 --
 -- TABLE pconfig
 --

diff --git a/doc/database.md b/doc/database.md
index 2ef77c6dad7e2a0a9b4608b5bbaf89a219b5b446..a0b28f4e84dbf331300cadeec7498bb1a57c1fa7 100644 (file)
--- a/doc/database.md
+++ b/doc/database.md
@@ -38,6 +38,8 @@ Database Tables
 | [manage](help/database/db_manage)                    | table of accounts that can "su" each other       |
 | [notify](help/database/db_notify)                    | notifications                                    |
 | [notify-threads](help/database/db_notify-threads)    |                                                  |
+| [oembed](help/database/db_oembed)                    | cache for OEmbed queries                         |
+| [parsed_url](help/database/db_parsed_url)            | cache for "parse_url" queries                    |
 | [pconfig](help/database/db_pconfig)                  | personal (per user) configuration storage        |
 | [photo](help/database/db_photo)                      | photo storage                                    |
 | [poll](help/database/db_poll)                        | data for polls                                   |

diff --git a/doc/database/db_oembed.md b/doc/database/db_oembed.md
new file mode 100644 (file)
index 0000000..5e994ec
--- /dev/null
+++ b/doc/database/db_oembed.md
@@ -0,0 +1,10 @@
+Table oembed
+============
+
+| Field        | Description                        | Type         | Null | Key | Default             | Extra |
+| ------------ | ---------------------------------- | ------------ | ---- | --- | ------------------- | ----- |
+| url          | page url                           | varchar(255) | NO   | PRI | NULL                |       |
+| content      | OEmbed data of the page            | text         | NO   |     | NULL                |       |
+| created      | datetime of creation               | datetime     | NO   | MUL | 0000-00-00 00:00:00 |       |
+
+Return to [database documentation](help/database)

diff --git a/doc/database/db_parsed_url.md b/doc/database/db_parsed_url.md
new file mode 100644 (file)
index 0000000..ada42c2
--- /dev/null
+++ b/doc/database/db_parsed_url.md
@@ -0,0 +1,12 @@
+Table parsed_url
+================
+
+| Field        | Description                        | Type         | Null | Key | Default             | Extra |
+| ------------ | ---------------------------------- | ------------ | ---- | --- | ------------------- | ----- |
+| url          | page url                           | varchar(255) | NO   | PRI | NULL                |       |
+| guessing     | is the "guessing" mode active?     | tinyint(1)   | NO   | PRI | 0                   |       |
+| oembed       | is the data the result of oembed?  | tinyint(1)   | NO   | PRI | 0                   |       |
+| content      | page data                          | text         | NO   |     | NULL                |       |
+| created      | datetime of creation               | datetime     | NO   | MUL | 0000-00-00 00:00:00 |       |
+
+Return to [database documentation](help/database)

diff --git a/include/Contact.php b/include/Contact.php
index c10bb9b791e08dc0a3befef65ef8613e9b94cad5..93d6237cbf421c826fec164b44e4fb96d1682f01 100644 (file)
--- a/include/Contact.php
+++ b/include/Contact.php
@@ -211,40 +211,40 @@ function get_contact_details_by_url($url, $uid = -1) {
        $r = q("SELECT `id`, `uid`, `url`, `network`, `name`, `nick`, `addr`, `location`, `about`, `keywords`, `gender`, `photo`, `addr`, `forum`, `prv`, `bd` FROM `contact` WHERE `nurl` = '%s' AND `uid` = %d AND `network` = '%s'",
                dbesc(normalise_link($url)), intval($uid), dbesc($profile["network"]));
 
-       if (!count($r))
+       if (!count($r) AND !isset($profile))
                $r = q("SELECT `id`, `uid`, `url`, `network`, `name`, `nick`, `addr`, `location`, `about`, `keywords`, `gender`, `photo`, `addr`, `forum`, `prv`, `bd` FROM `contact` WHERE `nurl` = '%s' AND `uid` = %d",
                        dbesc(normalise_link($url)), intval($uid));
 
-       if (!count($r))
+       if (!count($r) AND !isset($profile))
                $r = q("SELECT `id`, `uid`, `url`, `network`, `name`, `nick`, `addr`, `location`, `about`, `keywords`, `gender`, `photo`, `addr`, `forum`, `prv`, `bd` FROM `contact` WHERE `nurl` = '%s' AND `uid` = 0",
                        dbesc(normalise_link($url)));
 
        if ($r) {
-               if (isset($r[0]["url"]) AND $r[0]["url"])
+               if (!isset($profile["url"]) AND $r[0]["url"])
                        $profile["url"] = $r[0]["url"];
-               if (isset($r[0]["name"]) AND $r[0]["name"])
+               if (!isset($profile["name"]) AND $r[0]["name"])
                        $profile["name"] = $r[0]["name"];
-               if (isset($r[0]["nick"]) AND $r[0]["nick"] AND ($profile["nick"] == ""))
+               if (!isset($profile["nick"]) AND $r[0]["nick"])
                        $profile["nick"] = $r[0]["nick"];
-               if (isset($r[0]["addr"]) AND $r[0]["addr"] AND ($profile["addr"] == ""))
+               if (!isset($profile["addr"]) AND $r[0]["addr"])
                        $profile["addr"] = $r[0]["addr"];
-               if (isset($r[0]["photo"]) AND $r[0]["photo"])
+               if (!isset($profile["photo"]) AND $r[0]["photo"])
                        $profile["photo"] = $r[0]["photo"];
-               if (isset($r[0]["location"]) AND $r[0]["location"])
+               if (!isset($profile["location"]) AND $r[0]["location"])
                        $profile["location"] = $r[0]["location"];
-               if (isset($r[0]["about"]) AND $r[0]["about"])
+               if (!isset($profile["about"]) AND $r[0]["about"])
                        $profile["about"] = $r[0]["about"];
-               if (isset($r[0]["keywords"]) AND $r[0]["keywords"])
+               if (!isset($profile["keywords"]) AND $r[0]["keywords"])
                        $profile["keywords"] = $r[0]["keywords"];
-               if (isset($r[0]["gender"]) AND $r[0]["gender"])
+               if (!isset($profile["gender"]) AND $r[0]["gender"])
                        $profile["gender"] = $r[0]["gender"];
                if (isset($r[0]["forum"]) OR isset($r[0]["prv"]))
                        $profile["community"] = ($r[0]["forum"] OR $r[0]["prv"]);
-               if (isset($r[0]["network"]) AND $r[0]["network"])
+               if (!isset($profile["network"]) AND $r[0]["network"])
                        $profile["network"] = $r[0]["network"];
-               if (isset($r[0]["addr"]) AND $r[0]["addr"])
+               if (!isset($profile["addr"]) AND $r[0]["addr"])
                        $profile["addr"] = $r[0]["addr"];
-               if (isset($r[0]["bd"]) AND $r[0]["bd"])
+               if (!isset($profile["bd"]) AND $r[0]["bd"])
                        $profile["bd"] = $r[0]["bd"];
                if ($r[0]["uid"] == 0)
                        $profile["cid"] = 0;

diff --git a/include/acl_selectors.php b/include/acl_selectors.php
index d5730a93a9f7fee45ab63aef5080b4e1b0402614..69181b7359bcb1ab68af4e84892782156771bf8c 100644 (file)
--- a/include/acl_selectors.php
+++ b/include/acl_selectors.php
@@ -20,7 +20,7 @@ function group_select($selname,$selclass,$preselected = false,$size = 4) {
 
        $o .= "<select name=\"{$selname}[]\" id=\"$selclass\" class=\"$selclass\" multiple=\"multiple\" size=\"$size\" >\r\n";
 
-       $r = q("SELECT * FROM `group` WHERE `deleted` = 0 AND `uid` = %d ORDER BY `name` ASC",
+       $r = q("SELECT `id`, `name` FROM `group` WHERE NOT `deleted` AND `uid` = %d ORDER BY `name` ASC",
                intval(local_user())
        );
 
@@ -309,7 +309,7 @@ function populate_acl($user = null, $show_jotnets = false) {
                $pubmail_enabled = false;
 
                if(! $mail_disabled) {
-                       $r = q("SELECT * FROM `mailacct` WHERE `uid` = %d AND `server` != '' LIMIT 1",
+                       $r = q("SELECT `pubmail` FROM `mailacct` WHERE `uid` = %d AND `server` != '' LIMIT 1",
                                intval(local_user())
                        );
                        if(count($r)) {

diff --git a/include/config.php b/include/config.php
index 05bcea6212d7769359febba441b6b79c36bfc161..b27274e9a946671f321062bf0d78827b6f8ac784 100644 (file)
--- a/include/config.php
+++ b/include/config.php
@@ -19,7 +19,7 @@ if(! function_exists('load_config')) {
 function load_config($family) {
        global $a;
 
-       $r = q("SELECT * FROM `config` WHERE `cat` = '%s'", dbesc($family));
+       $r = q("SELECT `v`, `k` FROM `config` WHERE `cat` = '%s'", dbesc($family));
        if(count($r)) {
                foreach($r as $rr) {
                        $k = $rr['k'];
@@ -170,7 +170,7 @@ function set_config($family,$key,$value) {
 if(! function_exists('load_pconfig')) {
 function load_pconfig($uid,$family) {
        global $a;
-       $r = q("SELECT * FROM `pconfig` WHERE `cat` = '%s' AND `uid` = %d",
+       $r = q("SELECT `v`,`k` FROM `pconfig` WHERE `cat` = '%s' AND `uid` = %d",
                dbesc($family),
                intval($uid)
        );

diff --git a/include/conversation.php b/include/conversation.php
index 7eae1e052e0523552f05f0f6ef64eec37215065a..6c33be84fb33971d1f9a167e7607e244487268c0 100644 (file)
--- a/include/conversation.php
+++ b/include/conversation.php
@@ -811,16 +811,16 @@ function best_link_url($item,&$sparkle,$ssl_state = false) {
        if((local_user()) && (local_user() == $item['uid'])) {
                if(isset($a->contacts) && x($a->contacts,$clean_url)) {
                        if($a->contacts[$clean_url]['network'] === NETWORK_DFRN) {
-                               $best_url = $a->get_baseurl($ssl_state) . '/redir/' . $a->contacts[$clean_url]['id'];
+                               $best_url = 'redir/'.$a->contacts[$clean_url]['id'];
                                $sparkle = true;
                        } else
                                $best_url = $a->contacts[$clean_url]['url'];
                }
        } elseif (local_user()) {
-               $r = q("SELECT `id`, `network` FROM `contact` WHERE `network` = '%s' AND `uid` = %d AND `nurl` = '%s'",
+               $r = q("SELECT `id` FROM `contact` WHERE `network` = '%s' AND `uid` = %d AND `nurl` = '%s' LIMIT 1",
                        dbesc(NETWORK_DFRN), intval(local_user()), dbesc(normalise_link($clean_url)));
                if ($r) {
-                       $best_url = $a->get_baseurl($ssl_state).'/redir/'.$r[0]['id'];
+                       $best_url = 'redir/'.$r[0]['id'];
                        $sparkle = true;
                }
        }
@@ -876,7 +876,7 @@ function item_photo_menu($item){
                if(local_user() && local_user() == $item['uid'] && link_compare($item['url'],$item['author-link'])) {
                        $cid = $item['contact-id'];
                } else {
-                       $r = q("SELECT `id`, `network` FROM `contact` WHERE `uid` = %d AND `nurl` = '%s' ORDER BY `uid` DESC LIMIT 1",
+                       $r = q("SELECT `id`, `network` FROM `contact` WHERE `uid` = %d AND `nurl` = '%s' LIMIT 1",
                                intval(local_user()), dbesc(normalise_link($item['author-link'])));
                        if ($r) {
                                $cid = $r[0]["id"];

diff --git a/include/cron.php b/include/cron.php
index 09d141702ec53a0f7d4d53e816f8c80359e7317b..bf3243cb60f9513bec0d6eeca5af44296a97f738 100644 (file)
--- a/include/cron.php
+++ b/include/cron.php
@@ -158,93 +158,11 @@ function cron_run(&$argv, &$argc){
                proc_run('php','include/expire.php');
        }
 
-       $last = get_config('system','cache_last_cleared');
-
-       if($last) {
-               $next = $last + (3600); // Once per hour
-               $clear_cache = ($next <= time());
-       } else
-               $clear_cache = true;
-
-       if ($clear_cache) {
-               // clear old cache
-               Cache::clear();
-
-               // clear old item cache files
-               clear_cache();
-
-               // clear cache for photos
-               clear_cache($a->get_basepath(), $a->get_basepath()."/photo");
-
-               // clear smarty cache
-               clear_cache($a->get_basepath()."/view/smarty3/compiled", $a->get_basepath()."/view/smarty3/compiled");
-
-               // clear cache for image proxy
-               if (!get_config("system", "proxy_disabled")) {
-                       clear_cache($a->get_basepath(), $a->get_basepath()."/proxy");
-
-                       $cachetime = get_config('system','proxy_cache_time');
-                       if (!$cachetime) $cachetime = PROXY_DEFAULT_TIME;
-
-                       q('DELETE FROM `photo` WHERE `uid` = 0 AND `resource-id` LIKE "pic:%%" AND `created` < NOW() - INTERVAL %d SECOND', $cachetime);
-               }
-
-               // Maximum table size in megabyte
-               $max_tablesize = intval(get_config('system','optimize_max_tablesize')) * 1000000;
-               if ($max_tablesize == 0)
-                       $max_tablesize = 100 * 1000000; // Default are 100 MB
-
-               // Minimum fragmentation level in percent
-               $fragmentation_level = intval(get_config('system','optimize_fragmentation')) / 100;
-               if ($fragmentation_level == 0)
-                       $fragmentation_level = 0.3; // Default value is 30%
-
-               // Optimize some tables that need to be optimized
-               $r = q("SHOW TABLE STATUS");
-               foreach($r as $table) {
-
-                       // Don't optimize tables that are too large
-                       if ($table["Data_length"] > $max_tablesize)
-                               continue;
-
-                       // Don't optimize empty tables
-                       if ($table["Data_length"] == 0)
-                               continue;
-
-                       // Calculate fragmentation
-                       $fragmentation = $table["Data_free"] / $table["Data_length"];
-
-                       logger("Table ".$table["Name"]." - Fragmentation level: ".round($fragmentation * 100, 2), LOGGER_DEBUG);
-
-                       // Don't optimize tables that needn't to be optimized
-                       if ($fragmentation < $fragmentation_level)
-                               continue;
-
-                       // So optimize it
-                       logger("Optimize Table ".$table["Name"], LOGGER_DEBUG);
-                       q("OPTIMIZE TABLE `%s`", dbesc($table["Name"]));
-               }
-
-               set_config('system','cache_last_cleared', time());
-       }
+       // Clear cache entries
+       cron_clear_cache($a);
 
-       // Repair missing Diaspora settings
-       $r = q("SELECT `id`, `url` FROM `contact`
-               WHERE `uid` > 0 AND `network` = '%s' AND (`batch` = '' OR `notify` = '' OR `poll` = '' OR pubkey = '')
-                       ORDER BY RAND() LIMIT 50", dbesc(NETWORK_DIASPORA));
-       if ($r) {
-               foreach ($r AS $contact) {
-                       if (poco_reachable($contact["url"])) {
-                               $data = probe_url($contact["url"]);
-                               if ($data["network"] == NETWORK_DIASPORA) {
-                                       logger("Repair contact ".$contact["id"]." ".$contact["url"], LOGGER_DEBUG);
-                                       q("UPDATE `contact` SET `batch` = '%s', `notify` = '%s', `poll` = '%s', pubkey = '%s' WHERE `id` = %d",
-                                               dbesc($data["batch"]), dbesc($data["notify"]), dbesc($data["poll"]), dbesc($data["pubkey"]),
-                                               intval($contact["id"]));
-                               }
-                       }
-               }
-       }
+       // Repair missing Diaspora values in contacts
+       cron_repair_diaspora($a);
 
        $manual_id  = 0;
        $generation = 0;
@@ -390,6 +308,115 @@ function cron_run(&$argv, &$argc){
        return;
 }
 
+/**
+ * @brief Clear cache entries
+ *
+ * @param App $a
+ */
+function cron_clear_cache(&$a) {
+
+       $last = get_config('system','cache_last_cleared');
+
+       if($last) {
+               $next = $last + (3600); // Once per hour
+               $clear_cache = ($next <= time());
+       } else
+               $clear_cache = true;
+
+       if (!$clear_cache)
+               return;
+
+       // clear old cache
+       Cache::clear();
+
+       // clear old item cache files
+       clear_cache();
+
+       // clear cache for photos
+       clear_cache($a->get_basepath(), $a->get_basepath()."/photo");
+
+       // clear smarty cache
+       clear_cache($a->get_basepath()."/view/smarty3/compiled", $a->get_basepath()."/view/smarty3/compiled");
+
+       // clear cache for image proxy
+       if (!get_config("system", "proxy_disabled")) {
+               clear_cache($a->get_basepath(), $a->get_basepath()."/proxy");
+
+               $cachetime = get_config('system','proxy_cache_time');
+               if (!$cachetime) $cachetime = PROXY_DEFAULT_TIME;
+
+               q('DELETE FROM `photo` WHERE `uid` = 0 AND `resource-id` LIKE "pic:%%" AND `created` < NOW() - INTERVAL %d SECOND', $cachetime);
+       }
+
+       // Delete the cached OEmbed entries that are older than one year
+       q("DELETE FROM `oembed` WHERE `created` < NOW() - INTERVAL 1 YEAR");
+
+       // Delete the cached "parse_url" entries that are older than one year
+       q("DELETE FROM `parsed_url` WHERE `created` < NOW() - INTERVAL 1 YEAR");
+
+       // Maximum table size in megabyte
+       $max_tablesize = intval(get_config('system','optimize_max_tablesize')) * 1000000;
+       if ($max_tablesize == 0)
+               $max_tablesize = 100 * 1000000; // Default are 100 MB
+
+       // Minimum fragmentation level in percent
+       $fragmentation_level = intval(get_config('system','optimize_fragmentation')) / 100;
+       if ($fragmentation_level == 0)
+               $fragmentation_level = 0.3; // Default value is 30%
+
+       // Optimize some tables that need to be optimized
+       $r = q("SHOW TABLE STATUS");
+       foreach($r as $table) {
+
+               // Don't optimize tables that are too large
+               if ($table["Data_length"] > $max_tablesize)
+                       continue;
+
+               // Don't optimize empty tables
+               if ($table["Data_length"] == 0)
+                       continue;
+
+               // Calculate fragmentation
+               $fragmentation = $table["Data_free"] / $table["Data_length"];
+
+               logger("Table ".$table["Name"]." - Fragmentation level: ".round($fragmentation * 100, 2), LOGGER_DEBUG);
+
+               // Don't optimize tables that needn't to be optimized
+               if ($fragmentation < $fragmentation_level)
+                       continue;
+
+               // So optimize it
+               logger("Optimize Table ".$table["Name"], LOGGER_DEBUG);
+               q("OPTIMIZE TABLE `%s`", dbesc($table["Name"]));
+       }
+
+       set_config('system','cache_last_cleared', time());
+}
+
+/**
+ * @brief Repair missing values in Diaspora contacts
+ *
+ * @param App $a
+ */
+function cron_repair_diaspora(&$a) {
+       $r = q("SELECT `id`, `url` FROM `contact`
+               WHERE `network` = '%s' AND (`batch` = '' OR `notify` = '' OR `poll` = '' OR pubkey = '')
+                       ORDER BY RAND() LIMIT 50", dbesc(NETWORK_DIASPORA));
+       if ($r) {
+               foreach ($r AS $contact) {
+                       if (poco_reachable($contact["url"])) {
+                               $data = probe_url($contact["url"]);
+                               if ($data["network"] == NETWORK_DIASPORA) {
+                                       logger("Repair contact ".$contact["id"]." ".$contact["url"], LOGGER_DEBUG);
+                                       q("UPDATE `contact` SET `batch` = '%s', `notify` = '%s', `poll` = '%s', pubkey = '%s' WHERE `id` = %d",
+                                               dbesc($data["batch"]), dbesc($data["notify"]), dbesc($data["poll"]), dbesc($data["pubkey"]),
+                                               intval($contact["id"]));
+                               }
+                       }
+               }
+       }
+}
+
 if (array_search(__file__,get_included_files())===0){
        cron_run($_SERVER["argv"],$_SERVER["argc"]);
        killme();

diff --git a/include/dbstructure.php b/include/dbstructure.php
index 7fed50e741c9192577201eee6370943295750200..43dfbd7b353185d9ad5f87ab19a82fbb7b061316 100644 (file)
--- a/include/dbstructure.php
+++ b/include/dbstructure.php
@@ -1021,6 +1021,30 @@ function db_definition() {
                                        "receiver-uid" => array("receiver-uid"),
                                        )
                        );
+       $database["oembed"] = array(
+                       "fields" => array(
+                                       "url" => array("type" => "varchar(255)", "not null" => "1", "primary" => "1"),
+                                       "content" => array("type" => "text", "not null" => "1"),
+                                       "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
+                                       ),
+                       "indexes" => array(
+                                       "PRIMARY" => array("url"),
+                                       "created" => array("created"),
+                                       )
+                       );
+       $database["parsed_url"] = array(
+                       "fields" => array(
+                                       "url" => array("type" => "varchar(255)", "not null" => "1", "primary" => "1"),
+                                       "guessing" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0", "primary" => "1"),
+                                       "oembed" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0", "primary" => "1"),
+                                       "content" => array("type" => "text", "not null" => "1"),
+                                       "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
+                                       ),
+                       "indexes" => array(
+                                       "PRIMARY" => array("url", "guessing", "oembed"),
+                                       "created" => array("created"),
+                                       )
+                       );
        $database["pconfig"] = array(
                        "fields" => array(
                                        "id" => array("type" => "int(11)", "not null" => "1", "extra" => "auto_increment", "primary" => "1"),

diff --git a/include/diaspora.php b/include/diaspora.php
index 1cf8897af9ea734c922eff3d8a86fea0601fb476..f03486fc211b0398bdb5f4c19614eda3a8a386d3 100644 (file)
--- a/include/diaspora.php
+++ b/include/diaspora.php
@@ -1973,11 +1973,15 @@ function diaspora_photo($importer,$xml,$msg,$attempt=1) {
                                           array($remote_photo_name, 'scaled_full_' . $remote_photo_name));
 
        if(strpos($parent_item['body'],$link_text) === false) {
+
+               $parent_item['body'] = $link_text . $parent_item['body'];
+
                $r = q("UPDATE `item` SET `body` = '%s', `visible` = 1 WHERE `id` = %d AND `uid` = %d",
-                       dbesc($link_text . $parent_item['body']),
+                       dbesc($parent_item['body']),
                        intval($parent_item['id']),
                        intval($parent_item['uid'])
                );
+               put_item_in_cache($parent_item, true);
                update_thread($parent_item['id']);
        }
 

diff --git a/include/items.php b/include/items.php
index 549027671d5e11c827c81ca43454a23acd0ee95f..a74ca3b36052ec826203124fc633daaee0a77956 100644 (file)
--- a/include/items.php
+++ b/include/items.php
@@ -955,8 +955,9 @@ function add_page_info_data($data) {
                $a = get_app();
                $hashtags = "\n";
                foreach ($data["keywords"] AS $keyword) {
-                       $hashtag = str_replace(array(" ", "+", "/", ".", "#", "'"),
-                                               array("","", "", "", "", ""), $keyword);
+                       /// @todo make a positive list of allowed characters
+                       $hashtag = str_replace(array(" ", "+", "/", ".", "#", "'", "’", "`", "(", ")", "„", "“"),
+                                               array("","", "", "", "", "", "", "", "", "", "", ""), $keyword);
                        $hashtags .= "#[url=".$a->get_baseurl()."/search?tag=".rawurlencode($hashtag)."]".$hashtag."[/url] ";
                }
        }
@@ -967,12 +968,7 @@ function add_page_info_data($data) {
 function query_page_info($url, $no_photos = false, $photo = "", $keywords = false, $keyword_blacklist = "") {
        require_once("mod/parse_url.php");
 
-       $data = Cache::get("parse_url:".$url);
-       if (is_null($data)){
-               $data = parseurl_getsiteinfo($url, true);
-               Cache::set("parse_url:".$url,serialize($data), CACHE_DAY);
-       } else
-               $data = unserialize($data);
+       $data = parseurl_getsiteinfo_cached($url, true);
 
        if ($photo != "")
                $data["images"][0]["src"] = $photo;

diff --git a/include/oembed.php b/include/oembed.php
index c848a4580c9dafc407c649fc2a36e0e2130eb399..b1770f68903408034053997dff8a0495081d89aa 100755 (executable)
--- a/include/oembed.php
+++ b/include/oembed.php
@@ -13,7 +13,13 @@ function oembed_fetch_url($embedurl, $no_rich_type = false){
 
        $a = get_app();
 
-       $txt = Cache::get($a->videowidth . $embedurl);
+       $r = q("SELECT * FROM `oembed` WHERE `url` = '%s'",
+               dbesc(normalise_link($embedurl)));
+
+       if ($r)
+               $txt = $r[0]["content"];
+       else
+               $txt = Cache::get($a->videowidth . $embedurl);
 
        // These media files should now be caught in bbcode.php
        // left here as a fallback in case this is called from another source
@@ -66,8 +72,14 @@ function oembed_fetch_url($embedurl, $no_rich_type = false){
 
                if ($txt[0]!="{")
                        $txt='{"type":"error"}';
-               else    //save in cache
+               else {  //save in cache
+                       $j = json_decode($txt);
+                       if ($j->type != "error")
+                               q("INSERT INTO `oembed` (`url`, `content`, `created`) VALUES ('%s', '%s', '%s')",
+                                       dbesc(normalise_link($embedurl)), dbesc($txt), dbesc(datetime_convert()));
+
                        Cache::set($a->videowidth . $embedurl,$txt, CACHE_DAY);
+               }
        }
 
        $j = json_decode($txt);
@@ -85,7 +97,7 @@ function oembed_fetch_url($embedurl, $no_rich_type = false){
        // If fetching information doesn't work, then improve via internal functions
        if (($j->type == "error") OR ($no_rich_type AND ($j->type == "rich"))) {
                require_once("mod/parse_url.php");
-               $data = parseurl_getsiteinfo($embedurl, true, false);
+               $data = parseurl_getsiteinfo_cached($embedurl, true, false);
                $j->type = $data["type"];
 
                if ($j->type == "photo") {

diff --git a/include/text.php b/include/text.php
index 1b03c39d26775b08fc4c30231bbab7bb956c7d59..fc5e79294f261bfbe1942b42a7daad69f46f3aeb 100644 (file)
--- a/include/text.php
+++ b/include/text.php
@@ -20,10 +20,10 @@ function replace_macros($s,$r) {
        $stamp1 = microtime(true);
 
        $a = get_app();
-       
+
        // pass $baseurl to all templates
        $r['$baseurl'] = z_root();
-       
+
 
        $t = $a->template_engine();
        try {
@@ -895,9 +895,9 @@ function contact_block() {
                $micropro = Null;
 
        } else {
-               $r = q("SELECT * FROM `contact`
-                               WHERE `uid` = %d AND `self` = 0 AND `blocked` = 0 and `pending` = 0
-                                       AND `hidden` = 0 AND `archive` = 0
+               $r = q("SELECT `id`, `uid`, `addr`, `url`, `name`, `micro`, `network` FROM `contact`
+                               WHERE `uid` = %d AND NOT `self` AND NOT `blocked` AND NOT `pending`
+                                       AND NOT `hidden` AND NOT `archive`
                                AND `network` IN ('%s', '%s', '%s') ORDER BY RAND() LIMIT %d",
                                intval($a->profile['uid']),
                                dbesc(NETWORK_DFRN),
@@ -1415,7 +1415,14 @@ function prepare_body(&$item,$attach = false, $preview = false) {
        $item['hashtags'] = $hashtags;
        $item['mentions'] = $mentions;
 
-       put_item_in_cache($item, true);
+       // Update the cached values if there is no "zrl=..." on the links
+       $update = (!local_user() and !remote_user() and ($item["uid"] == 0));
+
+       // Or update it if the current viewer is the intented viewer
+       if (($item["uid"] == local_user()) AND ($item["uid"] != 0))
+               $update = true;
+
+       put_item_in_cache($item, $update);
        $s = $item["rendered-html"];
 
        $prep_arr = array('item' => $item, 'html' => $s, 'preview' => $preview);

diff --git a/include/threads.php b/include/threads.php
index dddcc4cdd311df4a328322fb72bbae6959d08eca..e542295d7b46e31bfeb2a067b9772633934b550a 100644 (file)
--- a/include/threads.php
+++ b/include/threads.php
@@ -112,7 +112,7 @@ function update_thread_uri($itemuri, $uid) {
 
 function update_thread($itemid, $setmention = false) {
        $items = q("SELECT `uid`, `guid`, `title`, `body`, `created`, `edited`, `commented`, `received`, `changed`, `wall`, `private`, `pubmail`, `moderated`, `visible`, `spam`, `starred`, `bookmark`, `contact-id`, `gcontact-id`,
-                       `deleted`, `origin`, `forum_mode`, `network`  FROM `item` WHERE `id` = %d AND (`parent` = %d OR `parent` = 0) LIMIT 1", intval($itemid), intval($itemid));
+                       `deleted`, `origin`, `forum_mode`, `network`, `rendered-html`, `rendered-hash` FROM `item` WHERE `id` = %d AND (`parent` = %d OR `parent` = 0) LIMIT 1", intval($itemid), intval($itemid));
 
        if (!$items)
                return;
@@ -125,7 +125,7 @@ function update_thread($itemid, $setmention = false) {
        $sql = "";
 
        foreach ($item AS $field => $data)
-               if (!in_array($field, array("guid", "title", "body"))) {
+               if (!in_array($field, array("guid", "title", "body", "rendered-html", "rendered-hash"))) {
                        if ($sql != "")
                                $sql .= ", ";
 
@@ -142,9 +142,11 @@ function update_thread($itemid, $setmention = false) {
        if (!$items)
                return;
 
-       $result = q("UPDATE `item` SET `title` = '%s', `body` = '%s' WHERE `id` = %d",
+       $result = q("UPDATE `item` SET `title` = '%s', `body` = '%s', `rendered-html` = '%s', `rendered-hash` = '%s' WHERE `id` = %d",
                        dbesc($item["title"]),
                        dbesc($item["body"]),
+                       dbesc($item["rendered-html"]),
+                       dbesc($item["rendered-hash"]),
                        intval($items[0]["id"])
                );
        logger("Updating public shadow for post ".$items[0]["id"]." - guid ".$item["guid"]." Result: ".print_r($result, true), LOGGER_DEBUG);

diff --git a/index.php b/index.php
index 89ed058465db00a7e7e9177e65632d95b6042b85..bf926d1fe76f166c7c240036560c86223d8f7814 100644 (file)
--- a/index.php
+++ b/index.php
@@ -556,6 +556,7 @@ EOT;
 $page    = $a->page;
 $profile = $a->profile;
 
+header("X-Friendica-Version: ".FRIENDICA_VERSION);
 header("Content-type: text/html; charset=utf-8");
 
 

diff --git a/mod/community.php b/mod/community.php
index 86bd18c869aab5fd238a842264d1eb5f921e5c30..b6d72a35555a64f375b1402fc3188c93357c3579 100644 (file)
--- a/mod/community.php
+++ b/mod/community.php
@@ -14,6 +14,10 @@ function community_content(&$a, $update = 0) {
 
        $o = '';
 
+       // Currently the community page isn't able to handle update requests
+       if ($update)
+               return;
+
        if((get_config('system','block_public')) && (! local_user()) && (! remote_user())) {
                notice( t('Public access denied.') . EOL);
                return;

diff --git a/mod/display.php b/mod/display.php
index c53e92d05b39829e7c37aae838340cee72ca4433..ca57af923332e8bad1d51534e1ed41deeb02249f 100644 (file)
--- a/mod/display.php
+++ b/mod/display.php
@@ -340,6 +340,10 @@ function display_content(&$a, $update = 0) {
                return;
        }
 
+       // Why do we need this on the display page? We don't have the possibility to write new content here.
+       // Ad editing of posts work without this as well.
+       // We should remove this completely for the 3.5.1 release.
+       /*
        if ($is_owner) {
                $x = array(
                        'is_owner' => true,
@@ -355,6 +359,7 @@ function display_content(&$a, $update = 0) {
                );
                $o .= status_editor($a,$x,0,true);
        }
+       */
 
        $sql_extra = item_permissions_sql($a->profile['uid'],$remote_contact,$groups);
 

diff --git a/mod/parse_url.php b/mod/parse_url.php
index 2eff09d252dd318eb53136aba857bfbd362479c1..a1ca5a3db5e2e1c831d35bada4b52a3502b54183 100644 (file)
--- a/mod/parse_url.php
+++ b/mod/parse_url.php
@@ -56,7 +56,15 @@ function completeurl($url, $scheme) {
 
 function parseurl_getsiteinfo_cached($url, $no_guessing = false, $do_oembed = true) {
 
-       $data = Cache::get("parse_url:".$no_guessing.":".$do_oembed.":".$url);
+       if ($url == "")
+               return false;
+
+       $r = q("SELECT * FROM `parsed_url` WHERE `url` = '%s' AND `guessing` = %d AND `oembed` = %d",
+               dbesc(normalise_link($url)), intval(!$no_guessing), intval($do_oembed));
+
+       if ($r)
+               $data = $r[0]["content"];
+
        if (!is_null($data)) {
                $data = unserialize($data);
                return $data;
@@ -64,7 +72,8 @@ function parseurl_getsiteinfo_cached($url, $no_guessing = false, $do_oembed = tr
 
        $data = parseurl_getsiteinfo($url, $no_guessing, $do_oembed);
 
-       Cache::set("parse_url:".$no_guessing.":".$do_oembed.":".$url,serialize($data), CACHE_DAY);
+       q("INSERT INTO `parsed_url` (`url`, `guessing`, `oembed`, `content`, `created`) VALUES ('%s', %d, %d, '%s', '%s')",
+               dbesc(normalise_link($url)), intval(!$no_guessing), intval($do_oembed), dbesc(serialize($data)), dbesc(datetime_convert()));
 
        return $data;
 }

diff --git a/update.php b/update.php
index 197e78de5c44f88f6262fd5e944f12a5f62c6aea..a825c82e9294657529aad6917f8d5cf56ce8a1ce 100644 (file)
--- a/update.php
+++ b/update.php
@@ -1,6 +1,6 @@
 <?php
 
-define( 'UPDATE_VERSION' , 1192 );
+define( 'UPDATE_VERSION' , 1193 );
 
 /**
  *

diff --git a/view/theme/vier/style.php b/view/theme/vier/style.php
index 72731a9bebc9843d3c7949c7a53e7a9abfe16593..3d3c776745de4c86a02527d334a0eab9a811b61f 100644 (file)
--- a/view/theme/vier/style.php
+++ b/view/theme/vier/style.php
@@ -1,12 +1,7 @@
 <?php
-
-if (file_exists("$THEMEPATH//style.css")){
-       echo file_get_contents("$THEMEPATH//style.css");
-}
-
 $uid = get_theme_uid();
 
-$style = get_pconfig( $uid, 'vier', 'style');
+$style = get_pconfig($uid, 'vier', 'style');
 
 if ($style == "")
        $style = get_config('vier', 'style');
@@ -15,14 +10,45 @@ if ($style == "")
        $style = "plus";
 
 if ($style == "flat")
-       $stylecss = file_get_contents('view/theme/vier/flat.css');
+       $stylecssfile = 'view/theme/vier/flat.css';
 else if ($style == "netcolour")
-       $stylecss = file_get_contents('view/theme/vier/netcolour.css');
+       $stylecssfile = 'view/theme/vier/netcolour.css';
 else if ($style == "breathe")
-       $stylecss = file_get_contents('view/theme/vier/breathe.css');
+       $stylecssfile = 'view/theme/vier/breathe.css';
 else if ($style == "plus")
-       $stylecss = file_get_contents('view/theme/vier/plus.css');
+       $stylecssfile = 'view/theme/vier/plus.css';
 else if ($style == "dark")
-       $stylecss = file_get_contents('view/theme/vier/dark.css');
+       $stylecssfile = 'view/theme/vier/dark.css';
+
+if (file_exists($THEMEPATH."//style.css")) {
+       $stylecss = file_get_contents($THEMEPATH."//style.css")."\n";
+       $modified = filemtime($THEMEPATH."//style.css");
+}
+
+$stylemodified = filemtime($stylecssfile);
+$stylecss .= file_get_contents($stylecssfile);
+
+if ($stylemodified > $modified)
+       $modified = $stylemodified;
 
+$modified = gmdate('r', $modified);
+
+$etag = md5($stylecss);
+
+// Only send the CSS file if it was changed
+header('Cache-Control: public');
+header('ETag: "'.$etag.'"');
+header('Last-Modified: '.$modified);
+
+if (isset($_SERVER['HTTP_IF_MODIFIED_SINCE']) || isset($_SERVER['HTTP_IF_NONE_MATCH'])) {
+
+       $cached_modified = gmdate('r', strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE']));
+       $cached_etag = str_replace(array('"', "-gzip"), array('', ''),
+                               stripslashes($_SERVER['HTTP_IF_NONE_MATCH']));
+
+       if (($cached_modified == $modified) AND ($cached_etag == $etag)) {
+               header('HTTP/1.1 304 Not Modified');
+               exit();
+       }
+}
 echo $stylecss;