X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;ds=sidebyside;f=classes%2FMemcached_DataObject.php;h=bc4c3a000ce11a732bc495e878fc018854dcd79d;hb=23eb49a017826621ea14f39b8fa55d839adfd8b8;hp=33645a3e8b6eeaf75246d69e87f70db41ab209a7;hpb=c8bc598cfd67353f33d7785556374b5d6865a7d9;p=quix0rs-gnu-social.git diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index 33645a3e8b..54c885e737 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -19,58 +19,8 @@ if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); } -class Memcached_DataObject extends DB_DataObject +class Memcached_DataObject extends Safe_DataObject { - /** - * Destructor to free global memory resources associated with - * this data object when it's unset or goes out of scope. - * DB_DataObject doesn't do this yet by itself. - */ - - function __destruct() - { - $this->free(); - if (method_exists('DB_DataObject', '__destruct')) { - parent::__destruct(); - } - } - - /** - * Magic function called at serialize() time. - * - * We use this to drop a couple process-specific references - * from DB_DataObject which can cause trouble in future - * processes. - * - * @return array of variable names to include in serialization. - */ - function __sleep() - { - $vars = array_keys(get_object_vars($this)); - $skip = array('_DB_resultid', '_link_loaded'); - return array_diff($vars, $skip); - } - - /** - * Magic function called at unserialize() time. - * - * Clean out some process-specific variables which might - * be floating around from a previous process's cached - * objects. - * - * Old cached objects may still have them. - */ - function __wakeup() - { - // Refers to global state info from a previous process. - // Clear this out so we don't accidentally break global - // state in *this* process. - $this->_DB_resultid = null; - - // We don't have any local DBO refs, so clear these out. - $this->_link_loaded = false; - } - /** * Wrapper for DB_DataObject's static lookup using memcached * as backing instead of an in-process cache array. @@ -84,16 +34,15 @@ class Memcached_DataObject extends DB_DataObject { if (is_null($v)) { $v = $k; - # XXX: HACK! - $i = new $cls; - $keys = $i->keys(); + $keys = self::pkeyCols($cls); + if (count($keys) > 1) { + // FIXME: maybe call pkeyGet() ourselves? + throw new Exception('Use pkeyGet() for compound primary keys'); + } $k = $keys[0]; - unset($i); } $i = Memcached_DataObject::getcached($cls, $k, $v); - if ($i) { - return $i; - } else { + if ($i === false) { // false == cache miss $i = DB_DataObject::factory($cls); if (empty($i)) { $i = false; @@ -101,29 +50,336 @@ class Memcached_DataObject extends DB_DataObject } $result = $i->get($k, $v); if ($result) { + // Hit! $i->encache(); - return $i; } else { + // save the fact that no such row exists + $c = self::memcache(); + if (!empty($c)) { + $ck = self::cachekey($cls, $k, $v); + $c->set($ck, null); + } $i = false; - return $i; } } + return $i; + } + + /** + * Get multiple items from the database by key + * + * @param string $cls Class to fetch + * @param string $keyCol name of column for key + * @param array $keyVals key values to fetch + * @param boolean $skipNulls return only non-null results? + * + * @return array Array of objects, in order + */ + function multiGet($cls, $keyCol, $keyVals, $skipNulls=true) + { + $result = self::pivotGet($cls, $keyCol, $keyVals); + + $values = array_values($result); + + if ($skipNulls) { + $tmp = array(); + foreach ($values as $value) { + if (!empty($value)) { + $tmp[] = $value; + } + } + $values = $tmp; + } + + return new ArrayWrapper($values); } - function &pkeyGet($cls, $kv) + /** + * Get multiple items from the database by key + * + * @param string $cls Class to fetch + * @param string $keyCol name of column for key + * @param array $keyVals key values to fetch + * @param boolean $otherCols Other columns to hold fixed + * + * @return array Array mapping $keyVals to objects, or null if not found + */ + static function pivotGet($cls, $keyCol, $keyVals, $otherCols = array()) + { + if (is_array($keyCol)) { + foreach ($keyVals as $keyVal) { + $result[implode(',', $keyVal)] = null; + } + } else { + $result = array_fill_keys($keyVals, null); + } + + $toFetch = array(); + + foreach ($keyVals as $keyVal) { + + if (is_array($keyCol)) { + $kv = array_combine($keyCol, $keyVal); + } else { + $kv = array($keyCol => $keyVal); + } + + $kv = array_merge($otherCols, $kv); + + $i = self::multicache($cls, $kv); + + if ($i !== false) { + if (is_array($keyCol)) { + $result[implode(',', $keyVal)] = $i; + } else { + $result[$keyVal] = $i; + } + } else if (!empty($keyVal)) { + $toFetch[] = $keyVal; + } + } + + if (count($toFetch) > 0) { + $i = DB_DataObject::factory($cls); + if (empty($i)) { + // TRANS: Exception thrown when a program code class (%s) cannot be instantiated. + throw new Exception(sprintf(_('Cannot instantiate class %s.'),$cls)); + } + foreach ($otherCols as $otherKeyCol => $otherKeyVal) { + $i->$otherKeyCol = $otherKeyVal; + } + if (is_array($keyCol)) { + $i->whereAdd(self::_inMultiKey($i, $keyCol, $toFetch)); + } else { + $i->whereAddIn($keyCol, $toFetch, $i->columnType($keyCol)); + } + if ($i->find()) { + while ($i->fetch()) { + $copy = clone($i); + $copy->encache(); + if (is_array($keyCol)) { + $vals = array(); + foreach ($keyCol as $k) { + $vals[] = $i->$k; + } + $result[implode(',', $vals)] = $copy; + } else { + $result[$i->$keyCol] = $copy; + } + } + } + + // Save state of DB misses + + foreach ($toFetch as $keyVal) { + $r = null; + if (is_array($keyCol)) { + $r = $result[implode(',', $keyVal)]; + } else { + $r = $result[$keyVal]; + } + if (empty($r)) { + if (is_array($keyCol)) { + $kv = array_combine($keyCol, $keyVal); + } else { + $kv = array($keyCol => $keyVal); + } + $kv = array_merge($otherCols, $kv); + // save the fact that no such row exists + $c = self::memcache(); + if (!empty($c)) { + $ck = self::multicacheKey($cls, $kv); + $c->set($ck, null); + } + } + } + } + + return $result; + } + + static function _inMultiKey($i, $cols, $values) + { + $types = array(); + + foreach ($cols as $col) { + $types[$col] = $i->columnType($col); + } + + $first = true; + + $query = ''; + + foreach ($values as $value) { + if ($first) { + $query .= '( '; + $first = false; + } else { + $query .= ' OR '; + } + $query .= '( '; + $i = 0; + $firstc = true; + foreach ($cols as $col) { + if (!$firstc) { + $query .= ' AND '; + } else { + $firstc = false; + } + switch ($types[$col]) { + case 'string': + case 'datetime': + $query .= sprintf("%s = %s", $col, $i->_quote($value[$i])); + break; + default: + $query .= sprintf("%s = %s", $col, $value[$i]); + break; + } + } + $query .= ') '; + } + + if (!$first) { + $query .= ' )'; + } + + return $query; + } + + static function pkeyCols($cls) + { + $i = DB_DataObject::factory($cls); + if (empty($i)) { + // TRANS: Exception thrown when a program code class (%s) cannot be instantiated. + throw new Exception(sprintf(_('Cannot instantiate class %s.'),$cls)); + } + $types = $i->keyTypes(); + ksort($types); + + $pkey = array(); + + foreach ($types as $key => $type) { + if ($type == 'K' || $type == 'N') { + $pkey[] = $key; + } + } + + return $pkey; + } + + function listGet($cls, $keyCol, $keyVals) + { + $pkeyMap = array_fill_keys($keyVals, array()); + $result = array_fill_keys($keyVals, array()); + + $pkeyCols = self::pkeyCols($cls); + + $toFetch = array(); + $allPkeys = array(); + + // We only cache keys -- not objects! + + foreach ($keyVals as $keyVal) { + $l = self::cacheGet(sprintf("%s:list-ids:%s:%s", strtolower($cls), $keyCol, $keyVal)); + if ($l !== false) { + $pkeyMap[$keyVal] = $l; + foreach ($l as $pkey) { + $allPkeys[] = $pkey; + } + } else { + $toFetch[] = $keyVal; + } + } + + if (count($allPkeys) > 0) { + $keyResults = self::pivotGet($cls, $pkeyCols, $allPkeys); + + foreach ($pkeyMap as $keyVal => $pkeyList) { + foreach ($pkeyList as $pkeyVal) { + $i = $keyResults[implode(',',$pkeyVal)]; + if (!empty($i)) { + $result[$keyVal][] = $i; + } + } + } + } + + if (count($toFetch) > 0) { + $i = DB_DataObject::factory($cls); + if (empty($i)) { + // TRANS: Exception thrown when a program code class (%s) cannot be instantiated. + throw new Exception(sprintf(_('Cannot instantiate class %s.'),$cls)); + } + $i->whereAddIn($keyCol, $toFetch, $i->columnType($keyCol)); + if ($i->find()) { + sprintf("listGet() got {$i->N} results for class $cls key $keyCol"); + while ($i->fetch()) { + $copy = clone($i); + $copy->encache(); + $result[$i->$keyCol][] = $copy; + $pkeyVal = array(); + foreach ($pkeyCols as $pkeyCol) { + $pkeyVal[] = $i->$pkeyCol; + } + $pkeyMap[$i->$keyCol][] = $pkeyVal; + } + } + foreach ($toFetch as $keyVal) { + self::cacheSet(sprintf("%s:list-ids:%s:%s", strtolower($cls), $keyCol, $keyVal), + $pkeyMap[$keyVal]); + } + } + + return $result; + } + + function columnType($columnName) + { + $keys = $this->table(); + if (!array_key_exists($columnName, $keys)) { + throw new Exception('Unknown key column ' . $columnName . ' in ' . join(',', array_keys($keys))); + } + + $def = $keys[$columnName]; + + if ($def & DB_DATAOBJECT_INT) { + return 'integer'; + } else { + return 'string'; + } + } + + /** + * @todo FIXME: Should this return false on lookup fail to match staticGet? + */ + function pkeyGet($cls, $kv) { $i = Memcached_DataObject::multicache($cls, $kv); - if ($i) { + if ($i !== false) { // false == cache miss return $i; } else { - $i = new $cls(); + $i = DB_DataObject::factory($cls); + if (empty($i) || PEAR::isError($i)) { + return false; + } foreach ($kv as $k => $v) { - $i->$k = $v; + if (is_null($v)) { + // XXX: possible SQL injection...? Don't + // pass keys from the browser, eh. + $i->whereAdd("$k is null"); + } else { + $i->$k = $v; + } } if ($i->find(true)) { $i->encache(); } else { $i = null; + $c = self::memcache(); + if (!empty($c)) { + $ck = self::multicacheKey($cls, $kv); + $c->set($ck, null); + } } return $i; } @@ -132,6 +388,10 @@ class Memcached_DataObject extends DB_DataObject function insert() { $result = parent::insert(); + if ($result) { + $this->fixupTimestamps(); + $this->encache(); // in case of cached negative lookups + } return $result; } @@ -142,6 +402,7 @@ class Memcached_DataObject extends DB_DataObject } $result = parent::update($orig); if ($result) { + $this->fixupTimestamps(); $this->encache(); } return $result; @@ -154,16 +415,17 @@ class Memcached_DataObject extends DB_DataObject } static function memcache() { - return common_memcache(); + return Cache::instance(); } static function cacheKey($cls, $k, $v) { - if (is_object($cls) || is_object($k) || is_object($v)) { + if (is_object($cls) || is_object($k) || (is_object($v) && !($v instanceof DB_DataObject_Cast))) { $e = new Exception(); common_log(LOG_ERR, __METHOD__ . ' object in param: ' . str_replace("\n", " ", $e->getTraceAsString())); } - return common_cache_key(strtolower($cls).':'.$k.':'.$v); + $vstr = self::valueString($v); + return Cache::key(strtolower($cls).':'.$k.':'.$vstr); } static function getcached($cls, $k, $v) { @@ -186,6 +448,17 @@ class Memcached_DataObject extends DB_DataObject function keyTypes() { + // ini-based classes return number-indexed arrays. handbuilt + // classes return column => keytype. Make this uniform. + + $keys = $this->keys(); + + $keyskeys = array_keys($keys); + + if (is_string($keyskeys[0])) { + return $keys; + } + global $_DB_DATAOBJECT; if (!isset($_DB_DATAOBJECT['INI'][$this->_database][$this->__table."__keys"])) { $this->databaseStructure(); @@ -197,6 +470,7 @@ class Memcached_DataObject extends DB_DataObject function encache() { $c = $this->memcache(); + if (!$c) { return false; } else if ($this->tableName() == 'user' && is_object($this->id)) { @@ -206,84 +480,107 @@ class Memcached_DataObject extends DB_DataObject str_replace("\n", " ", $e->getTraceAsString())); return false; } else { - $pkey = array(); - $pval = array(); - $types = $this->keyTypes(); - ksort($types); - foreach ($types as $key => $type) { - if ($type == 'K') { - $pkey[] = $key; - $pval[] = $this->$key; - } else { - $c->set($this->cacheKey($this->tableName(), $key, $this->$key), $this); - } + $keys = $this->_allCacheKeys(); + + foreach ($keys as $key) { + $c->set($key, $this); } - # XXX: should work for both compound and scalar pkeys - $pvals = implode(',', $pval); - $pkeys = implode(',', $pkey); - $c->set($this->cacheKey($this->tableName(), $pkeys, $pvals), $this); } } function decache() { $c = $this->memcache(); + if (!$c) { return false; - } else { - $pkey = array(); - $pval = array(); - $types = $this->keyTypes(); - ksort($types); - foreach ($types as $key => $type) { - if ($type == 'K') { - $pkey[] = $key; - $pval[] = $this->$key; - } else { - $c->delete($this->cacheKey($this->tableName(), $key, $this->$key)); + } + + $keys = $this->_allCacheKeys(); + + foreach ($keys as $key) { + $c->delete($key, $this); + } + } + + function _allCacheKeys() + { + $ckeys = array(); + + $types = $this->keyTypes(); + ksort($types); + + $pkey = array(); + $pval = array(); + + foreach ($types as $key => $type) { + + assert(!empty($key)); + + if ($type == 'U') { + if (empty($this->$key)) { + continue; } + $ckeys[] = $this->cacheKey($this->tableName(), $key, self::valueString($this->$key)); + } else if ($type == 'K' || $type == 'N') { + $pkey[] = $key; + $pval[] = self::valueString($this->$key); + } else { + // Low level exception. No need for i18n as discussed with Brion. + throw new Exception("Unknown key type $key => $type for " . $this->tableName()); } - # should work for both compound and scalar pkeys - # XXX: comma works for now but may not be safe separator for future keys - $pvals = implode(',', $pval); - $pkeys = implode(',', $pkey); - $c->delete($this->cacheKey($this->tableName(), $pkeys, $pvals)); } + + assert(count($pkey) > 0); + + // XXX: should work for both compound and scalar pkeys + $pvals = implode(',', $pval); + $pkeys = implode(',', $pkey); + + $ckeys[] = $this->cacheKey($this->tableName(), $pkeys, $pvals); + + return $ckeys; } function multicache($cls, $kv) { ksort($kv); - $c = Memcached_DataObject::memcache(); + $c = self::memcache(); if (!$c) { return false; } else { - $pkeys = implode(',', array_keys($kv)); - $pvals = implode(',', array_values($kv)); - return $c->get(Memcached_DataObject::cacheKey($cls, $pkeys, $pvals)); + return $c->get(self::multicacheKey($cls, $kv)); } } + static function multicacheKey($cls, $kv) + { + ksort($kv); + $pkeys = implode(',', array_keys($kv)); + $pvals = implode(',', array_values($kv)); + return self::cacheKey($cls, $pkeys, $pvals); + } + function getSearchEngine($table) { require_once INSTALLDIR.'/lib/search_engines.php'; - static $search_engine; - if (!isset($search_engine)) { - if (Event::handle('GetSearchEngine', array($this, $table, &$search_engine))) { - if ('mysql' === common_config('db', 'type')) { - $type = common_config('search', 'type'); - if ($type == 'like') { - $search_engine = new MySQLLikeSearch($this, $table); - } else if ($type == 'fulltext') { - $search_engine = new MySQLSearch($this, $table); - } else { - throw new ServerException('Unknown search type: ' . $type); - } + + if (Event::handle('GetSearchEngine', array($this, $table, &$search_engine))) { + if ('mysql' === common_config('db', 'type')) { + $type = common_config('search', 'type'); + if ($type == 'like') { + $search_engine = new MySQLLikeSearch($this, $table); + } else if ($type == 'fulltext') { + $search_engine = new MySQLSearch($this, $table); } else { - $search_engine = new PGSearch($this, $table); + // Low level exception. No need for i18n as discussed with Brion. + throw new ServerException('Unknown search type: ' . $type); } + } else { + $search_engine = new PGSearch($this, $table); } } + return $search_engine; } @@ -295,10 +592,11 @@ class Memcached_DataObject extends DB_DataObject $inst->query($qry); return $inst; } - $key_part = common_keyize($cls).':'.md5($qry); - $ckey = common_cache_key($key_part); + $key_part = Cache::keyize($cls).':'.md5($qry); + $ckey = Cache::key($key_part); $stored = $c->get($ckey); - if ($stored) { + + if ($stored !== false) { return new ArrayWrapper($stored); } @@ -309,12 +607,12 @@ class Memcached_DataObject extends DB_DataObject $cached[] = clone($inst); } $inst->free(); - $c->set($ckey, $cached, MEMCACHE_COMPRESSED, $expiry); + $c->set($ckey, $cached, Cache::COMPRESSED, $expiry); return new ArrayWrapper($cached); } /** - * sends query to database - this is the private one that must work + * sends query to database - this is the private one that must work * - internal functions use this rather than $this->query() * * Overridden to do logging. @@ -325,18 +623,105 @@ class Memcached_DataObject extends DB_DataObject */ function _query($string) { + if (common_config('db', 'annotate_queries')) { + $string = $this->annotateQuery($string); + } + $start = microtime(true); - $result = parent::_query($string); + $fail = false; + $result = null; + if (Event::handle('StartDBQuery', array($this, $string, &$result))) { + common_perf_counter('query', $string); + try { + $result = parent::_query($string); + } catch (Exception $e) { + $fail = $e; + } + Event::handle('EndDBQuery', array($this, $string, &$result)); + } $delta = microtime(true) - $start; $limit = common_config('db', 'log_slow_queries'); if (($limit > 0 && $delta >= $limit) || common_config('db', 'log_queries')) { $clean = $this->sanitizeQuery($string); - common_log(LOG_DEBUG, sprintf("DB query (%0.3fs): %s", $delta, $clean)); + if ($fail) { + $msg = sprintf("FAILED DB query (%0.3fs): %s - %s", $delta, $fail->getMessage(), $clean); + } else { + $msg = sprintf("DB query (%0.3fs): %s", $delta, $clean); + } + common_log(LOG_DEBUG, $msg); + } + + if ($fail) { + throw $fail; } return $result; } + /** + * Find the first caller in the stack trace that's not a + * low-level database function and add a comment to the + * query string. This should then be visible in process lists + * and slow query logs, to help identify problem areas. + * + * Also marks whether this was a web GET/POST or which daemon + * was running it. + * + * @param string $string SQL query string + * @return string SQL query string, with a comment in it + */ + function annotateQuery($string) + { + $ignore = array('annotateQuery', + '_query', + 'query', + 'get', + 'insert', + 'delete', + 'update', + 'find'); + $ignoreStatic = array('staticGet', + 'pkeyGet', + 'cachedQuery'); + $here = get_class($this); // if we get confused + $bt = debug_backtrace(); + + // Find the first caller that's not us? + foreach ($bt as $frame) { + $func = $frame['function']; + if (isset($frame['type']) && $frame['type'] == '::') { + if (in_array($func, $ignoreStatic)) { + continue; + } + $here = $frame['class'] . '::' . $func; + break; + } else if (isset($frame['type']) && $frame['type'] == '->') { + if ($frame['object'] === $this && in_array($func, $ignore)) { + continue; + } + if (in_array($func, $ignoreStatic)) { + continue; // @todo FIXME: This shouldn't be needed? + } + $here = get_class($frame['object']) . '->' . $func; + break; + } + $here = $func; + break; + } + + if (php_sapi_name() == 'cli') { + $context = basename($_SERVER['PHP_SELF']); + } else { + $context = $_SERVER['REQUEST_METHOD']; + } + + // Slip the comment in after the first command, + // or DB_DataObject gets confused about handling inserts and such. + $parts = explode(' ', $string, 2); + $parts[0] .= " /* $context $here */"; + return implode(' ', $parts); + } + // Sanitize a query for logging // @fixme don't trim spaces in string literals function sanitizeQuery($string) @@ -376,7 +761,7 @@ class Memcached_DataObject extends DB_DataObject // // WARNING WARNING if we end up actually using multiple DBs at a time // we'll need some fancier logic here. - if (!$exists && !empty($_DB_DATAOBJECT['CONNECTIONS'])) { + if (!$exists && !empty($_DB_DATAOBJECT['CONNECTIONS']) && php_sapi_name() == 'cli') { foreach ($_DB_DATAOBJECT['CONNECTIONS'] as $index => $conn) { if (!empty($conn)) { $conn->disconnect(); @@ -400,6 +785,10 @@ class Memcached_DataObject extends DB_DataObject } } } + // Needed to make timestamp values usefully comparable. + if (common_config('db', 'type') == 'mysql') { + parent::_query("set time_zone='+0:00'"); + } } return $result; @@ -453,9 +842,111 @@ class Memcached_DataObject extends DB_DataObject } if (!$dsn) { - throw new Exception("No database name / dsn found anywhere"); + // TRANS: Exception thrown when database name or Data Source Name could not be found. + throw new Exception(_('No database name or DSN found anywhere.')); } return $dsn; } + + static function blow() + { + $c = self::memcache(); + + if (empty($c)) { + return false; + } + + $args = func_get_args(); + + $format = array_shift($args); + + $keyPart = vsprintf($format, $args); + + $cacheKey = Cache::key($keyPart); + + return $c->delete($cacheKey); + } + + function fixupTimestamps() + { + // Fake up timestamp columns + $columns = $this->table(); + foreach ($columns as $name => $type) { + if ($type & DB_DATAOBJECT_MYSQLTIMESTAMP) { + $this->$name = common_sql_now(); + } + } + } + + function debugDump() + { + common_debug("debugDump: " . common_log_objstring($this)); + } + + function raiseError($message, $type = null, $behaviour = null) + { + $id = get_class($this); + if (!empty($this->id)) { + $id .= ':' . $this->id; + } + if ($message instanceof PEAR_Error) { + $message = $message->getMessage(); + } + // Low level exception. No need for i18n as discussed with Brion. + throw new ServerException("[$id] DB_DataObject error [$type]: $message"); + } + + static function cacheGet($keyPart) + { + $c = self::memcache(); + + if (empty($c)) { + return false; + } + + $cacheKey = Cache::key($keyPart); + + return $c->get($cacheKey); + } + + static function cacheSet($keyPart, $value, $flag=null, $expiry=null) + { + $c = self::memcache(); + + if (empty($c)) { + return false; + } + + $cacheKey = Cache::key($keyPart); + + return $c->set($cacheKey, $value, $flag, $expiry); + } + + static function valueString($v) + { + $vstr = null; + if (is_object($v) && $v instanceof DB_DataObject_Cast) { + switch ($v->type) { + case 'date': + $vstr = $v->year . '-' . $v->month . '-' . $v->day; + break; + case 'blob': + case 'string': + case 'sql': + case 'datetime': + case 'time': + // Low level exception. No need for i18n as discussed with Brion. + throw new ServerException("Unhandled DB_DataObject_Cast type passed as cacheKey value: '$v->type'"); + break; + default: + // Low level exception. No need for i18n as discussed with Brion. + throw new ServerException("Unknown DB_DataObject_Cast type passed as cacheKey value: '$v->type'"); + break; + } + } else { + $vstr = strval($v); + } + return $vstr; + } }