From: Philipp Holzer Date: Thu, 28 Jun 2018 20:57:17 +0000 (+0200) Subject: redesign of locking & caching X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=3f7e4f5bb67c67586423d9e1105ce274b83767c3;p=friendica.git redesign of locking & caching - New Factory "CacheDriverFactory" for Cache and Locks - Adding Redis/Memcached Locking - Moved Lock to Core - other improvements --- diff --git a/src/Core/Cache.php b/src/Core/Cache.php index 4202db325c..cc77d9bfd0 100644 --- a/src/Core/Cache.php +++ b/src/Core/Cache.php @@ -4,8 +4,7 @@ */ namespace Friendica\Core; -use Friendica\Core\Cache; -use Friendica\Core\Config; +use Friendica\Core\Cache\CacheDriverFactory; /** * @brief Class for storing data for a short time @@ -24,31 +23,13 @@ class Cache extends \Friendica\BaseObject /** * @var Cache\ICacheDriver */ - static $driver = null; + private static $driver = null; public static function init() { - switch(Config::get('system', 'cache_driver', 'database')) { - case 'memcache': - $memcache_host = Config::get('system', 'memcache_host', '127.0.0.1'); - $memcache_port = Config::get('system', 'memcache_port', 11211); - - self::$driver = new Cache\MemcacheCacheDriver($memcache_host, $memcache_port); - break; - case 'memcached': - $memcached_hosts = Config::get('system', 'memcached_hosts', [['127.0.0.1', 11211]]); - - self::$driver = new Cache\MemcachedCacheDriver($memcached_hosts); - break; - case 'redis': - $redis_host = Config::get('system', 'redis_host', '127.0.0.1'); - $redis_port = Config::get('system', 'redis_port', 6379); - - self::$driver = new Cache\RedisCacheDriver($redis_host, $redis_port); - break; - default: - self::$driver = new Cache\DatabaseCacheDriver(); - } + $driver_name = Config::get('system', 'cache_driver', 'database'); + + self::$driver = CacheDriverFactory::create($driver_name); } /** diff --git a/src/Core/Cache/CacheDriverFactory.php b/src/Core/Cache/CacheDriverFactory.php new file mode 100644 index 0000000000..d15b8e9858 --- /dev/null +++ b/src/Core/Cache/CacheDriverFactory.php @@ -0,0 +1,48 @@ +getMessage()); + } + } + + // 2. Try to use Cache Locking (don't use the DB-Cache Locking because it works different!) + $cache_driver = Config::get('system', 'cache_driver', 'database'); + if ($cache_driver != 'database') { + try { + $lock_driver = CacheDriverFactory::create($cache_driver); + self::$driver = new Lock\CacheLockDriver($lock_driver); + return; + } catch (\Exception $exception) { + logger('Using Cache driver for locking failed: ' . $exception->getMessage()); + } + } + + // 3. Use Database Locking as a Fallback + self::$driver = new Lock\DatabaseLockDriver(); + } + + /** + * Returns the current cache driver + * + * @return Lock\ILockDriver; + */ + private static function getDriver() + { + if (self::$driver === null) { + self::init(); + } + + return self::$driver; + } + + /** + * @brief Acquires a lock for a given name + * + * @param string $key Name of the lock + * @param integer $timeout Seconds until we give up + * + * @return boolean Was the lock successful? + */ + public static function acquireLock($key, $timeout = 120) + { + return self::getDriver()->acquireLock($key, $timeout); + } + + /** + * @brief Releases a lock if it was set by us + * + * @param string $key Name of the lock + * @return mixed + */ + public static function releaseLock($key) + { + return self::getDriver()->releaseLock($key); + } + + /** + * @brief Releases all lock that were set by us + * @return void + */ + public static function releaseAll() + { + self::getDriver()->releaseAll(); + } +} diff --git a/src/Core/Lock/AbstractLockDriver.php b/src/Core/Lock/AbstractLockDriver.php new file mode 100644 index 0000000000..15820c3782 --- /dev/null +++ b/src/Core/Lock/AbstractLockDriver.php @@ -0,0 +1,57 @@ +acquireLock[$key]); + } + + /** + * @brief Mark a locally acquired lock + * + * @param string $key The Name of the lock + */ + protected function markAcquire(string $key) { + $this->acquiredLocks[$key] = true; + } + + /** + * @brief Mark a release of a locally acquired lock + * + * @param string $key The Name of the lock + */ + protected function markRelease(string $key) { + unset($this->acquiredLocks[$key]); + } + + /** + * @brief Releases all lock that were set by us + * + * @return void + */ + public function releaseAll() { + foreach ($this->acquiredLocks as $acquiredLock) { + $this->releaseLock($acquiredLock); + } + } +} diff --git a/src/Core/Lock/CacheLockDriver.php b/src/Core/Lock/CacheLockDriver.php new file mode 100644 index 0000000000..0adca140d1 --- /dev/null +++ b/src/Core/Lock/CacheLockDriver.php @@ -0,0 +1,89 @@ +cache = $cache; + } + + /** + * + * @brief Sets a lock for a given name + * + * @param string $key The Name of the lock + * @param integer $timeout Seconds until we give up + * + * @return boolean Was the lock successful? + */ + public function acquireLock(string $key, int $timeout = 120) + { + $got_lock = false; + $start = time(); + + $cachekey = get_app()->get_hostname() . ";lock:" . $key; + + do { + $lock = $this->cache->get($cachekey); + + if (!is_bool($lock)) { + $pid = (int)$lock; + + // When the process id isn't used anymore, we can safely claim the lock for us. + // Or we do want to lock something that was already locked by us. + if (!posix_kill($pid, 0) || ($pid == getmypid())) { + $lock = false; + } + } + if (is_bool($lock)) { + $this->cache->set($cachekey, getmypid(), 300); + $got_lock = true; + } + + if (!$got_lock && ($timeout > 0)) { + usleep(rand(10000, 200000)); + } + } while (!$got_lock && ((time() - $start) < $timeout)); + + $this->markAcquire($key); + + return $got_lock; + } + + /** + * @brief Removes a lock if it was set by us + * + * @param string $key Name of the lock + * + * @return mixed + */ + public function releaseLock(string $key) + { + $cachekey = get_app()->get_hostname() . ";lock:" . $key; + $lock = $this->cache->get($cachekey); + + if (!is_bool($lock)) { + if ((int)$lock == getmypid()) { + $this->cache->delete($cachekey); + } + } + + $this->markRelease($key); + + return; + } +} diff --git a/src/Core/Lock/DatabaseLockDriver.php b/src/Core/Lock/DatabaseLockDriver.php new file mode 100644 index 0000000000..f9878340cf --- /dev/null +++ b/src/Core/Lock/DatabaseLockDriver.php @@ -0,0 +1,89 @@ + $key]); + + if (DBM::is_result($lock)) { + if ($lock['locked']) { + // When the process id isn't used anymore, we can safely claim the lock for us. + if (!posix_kill($lock['pid'], 0)) { + $lock['locked'] = false; + } + // We want to lock something that was already locked by us? So we got the lock. + if ($lock['pid'] == getmypid()) { + $got_lock = true; + } + } + if (!$lock['locked']) { + dba::update('locks', ['locked' => true, 'pid' => getmypid()], ['name' => $key]); + $got_lock = true; + } + } else { + dba::insert('locks', ['name' => $key, 'locked' => true, 'pid' => getmypid()]); + $got_lock = true; + } + + dba::unlock(); + + if (!$got_lock && ($timeout > 0)) { + usleep(rand(100000, 2000000)); + } + } while (!$got_lock && ((time() - $start) < $timeout)); + + $this->markAcquire($key); + + return $got_lock; + } + + /** + * @brief Removes a lock if it was set by us + * + * @param string $key Name of the lock + * + * @return mixed + */ + public function releaseLock(string $key) + { + dba::delete('locks', ['locked' => false, 'pid' => 0], ['name' => $key, 'pid' => getmypid()]); + + $this->releaseLock($key); + + return; + } + + /** + * @brief Removes all lock that were set by us + * + * @return void + */ + public function releaseAll() + { + dba::delete('locks', ['locked' => false, 'pid' => 0], ['pid' => getmypid()]); + + $this->acquiredLocks = []; + } +} diff --git a/src/Core/Lock/ILockDriver.php b/src/Core/Lock/ILockDriver.php new file mode 100644 index 0000000000..39e4ba8e89 --- /dev/null +++ b/src/Core/Lock/ILockDriver.php @@ -0,0 +1,38 @@ + + */ +interface ILockDriver +{ + /** + * + * @brief Acquires a lock for a given name + * + * @param string $key The Name of the lock + * @param integer $timeout Seconds until we give up + * + * @return boolean Was the lock successful? + */ + public function acquireLock(string $key, int $timeout = 120); + + /** + * @brief Releases a lock if it was set by us + * + * @param string $key The Name of the lock + * + * @return void + */ + public function releaseLock(string $key); + + /** + * @brief Releases all lock that were set by us + * + * @return void + */ + public function releaseAll(); +} diff --git a/src/Core/Lock/SemaphoreLockDriver.php b/src/Core/Lock/SemaphoreLockDriver.php new file mode 100644 index 0000000000..84965a1643 --- /dev/null +++ b/src/Core/Lock/SemaphoreLockDriver.php @@ -0,0 +1,68 @@ +acquiredLocks[$key] = sem_get(self::semaphoreKey($key)); + if ($this->acquiredLocks[$key]) { + return sem_acquire($this->acquiredLocks[$key], ($timeout == 0)); + } + } + + /** + * @brief Removes a lock if it was set by us + * + * @param string $key Name of the lock + * + * @return mixed + */ + public function releaseLock(string $key) + { + if (empty($this->acquiredLocks[$key])) { + return false; + } else { + $success = @sem_release($this->acquiredLocks[$key]); + unset($this->acquiredLocks[$key]); + return $success; + } + } +} diff --git a/src/Core/Worker.php b/src/Core/Worker.php index ef7873fcd4..cbf2ae8bd9 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -7,10 +7,10 @@ namespace Friendica\Core; use Friendica\Core\Addon; use Friendica\Core\Config; use Friendica\Core\System; +use Friendica\Core\Lock; use Friendica\Database\DBM; use Friendica\Model\Process; use Friendica\Util\DateTimeFormat; -use Friendica\Util\Lock; use Friendica\Util\Network; use dba; diff --git a/src/Protocol/OStatus.php b/src/Protocol/OStatus.php index a1223b7b68..288cbfed1b 100644 --- a/src/Protocol/OStatus.php +++ b/src/Protocol/OStatus.php @@ -9,6 +9,7 @@ use Friendica\Content\Text\HTML; use Friendica\Core\Cache; use Friendica\Core\Config; use Friendica\Core\L10n; +use Friendica\Core\Lock; use Friendica\Core\System; use Friendica\Database\DBM; use Friendica\Model\Contact; @@ -19,7 +20,6 @@ use Friendica\Model\User; use Friendica\Network\Probe; use Friendica\Object\Image; use Friendica\Util\DateTimeFormat; -use Friendica\Util\Lock; use Friendica\Util\Network; use Friendica\Util\XML; use dba; diff --git a/src/Util/Lock.php b/src/Util/Lock.php deleted file mode 100644 index b6a4a97f18..0000000000 --- a/src/Util/Lock.php +++ /dev/null @@ -1,97 +0,0 @@ -=')) { - self::$driver = new Lock\SemaphoreLockDriver(); - } elseif (Config::get('system', 'cache_driver', 'database') == 'memcache') { - self::$driver = new Lock\MemcacheLockDriver(); - } else { - self::$driver = new Lock\DatabaseLockDriver(); - } - } - } - - /** - * Returns the current cache driver - * - * @return Lock\ILockDriver; - */ - private static function getDriver() - { - if (self::$driver === null) { - self::init(); - } - - return self::$driver; - } - - /** - * @brief Acquires a lock for a given name - * - * @param string $key Name of the lock - * @param integer $timeout Seconds until we give up - * - * @return boolean Was the lock successful? - */ - public static function acquireLock($key, $timeout = 120) - { - return self::getDriver()->acquireLock($key, $timeout); - } - - /** - * @brief Releases a lock if it was set by us - * - * @param string $key Name of the lock - * @return mixed - */ - public static function releaseLock($key) - { - return self::getDriver()->releaseLock($key); - } - - /** - * @brief Releases all lock that were set by us - * @return void - */ - public static function releaseAll() - { - self::getDriver()->releaseAll(); - } -} diff --git a/src/Util/Lock/DatabaseLockDriver.php b/src/Util/Lock/DatabaseLockDriver.php deleted file mode 100644 index b2e8f50279..0000000000 --- a/src/Util/Lock/DatabaseLockDriver.php +++ /dev/null @@ -1,83 +0,0 @@ - $key]); - - if (DBM::is_result($lock)) { - if ($lock['locked']) { - // When the process id isn't used anymore, we can safely claim the lock for us. - if (!posix_kill($lock['pid'], 0)) { - $lock['locked'] = false; - } - // We want to lock something that was already locked by us? So we got the lock. - if ($lock['pid'] == getmypid()) { - $got_lock = true; - } - } - if (!$lock['locked']) { - dba::update('locks', ['locked' => true, 'pid' => getmypid()], ['name' => $key]); - $got_lock = true; - } - } elseif (!DBM::is_result($lock)) { - dba::insert('locks', ['name' => $key, 'locked' => true, 'pid' => getmypid()]); - $got_lock = true; - } - - dba::unlock(); - - if (!$got_lock && ($timeout > 0)) { - usleep(rand(100000, 2000000)); - } - } while (!$got_lock && ((time() - $start) < $timeout)); - - return $got_lock; - } - - /** - * @brief Removes a lock if it was set by us - * - * @param string $key Name of the lock - * - * @return mixed - */ - public function releaseLock($key) - { - dba::update('locks', ['locked' => false, 'pid' => 0], ['name' => $key, 'pid' => getmypid()]); - - return; - } - - /** - * @brief Removes all lock that were set by us - * - * @return void - */ - public function releaseAll() - { - dba::update('locks', ['locked' => false, 'pid' => 0], ['pid' => getmypid()]); - } -} diff --git a/src/Util/Lock/ILockDriver.php b/src/Util/Lock/ILockDriver.php deleted file mode 100644 index 7f5a359946..0000000000 --- a/src/Util/Lock/ILockDriver.php +++ /dev/null @@ -1,38 +0,0 @@ - - */ -interface ILockDriver -{ - /** - * - * @brief Acquires a lock for a given name - * - * @param string $key The Name of the lock - * @param integer $timeout Seconds until we give up - * - * @return boolean Was the lock successful? - */ - public function acquireLock($key, $timeout = 120); - - /** - * @brief Releases a lock if it was set by us - * - * @param string $key Name of the lock - * - * @return mixed - */ - public function releaseLock($key); - - /** - * @brief Releases all lock that were set by us - * - * @return void - */ - public function releaseAll(); -} diff --git a/src/Util/Lock/MemcacheLockDriver.php b/src/Util/Lock/MemcacheLockDriver.php deleted file mode 100644 index d0789a95c4..0000000000 --- a/src/Util/Lock/MemcacheLockDriver.php +++ /dev/null @@ -1,86 +0,0 @@ -get_hostname() . ";lock:" . $key; - - do { - // We only lock to be sure that nothing happens at exactly the same time - dba::lock('locks'); - $lock = Cache::get($cachekey); - - if (!is_bool($lock)) { - $pid = (int)$lock; - - // When the process id isn't used anymore, we can safely claim the lock for us. - // Or we do want to lock something that was already locked by us. - if (!posix_kill($pid, 0) || ($pid == getmypid())) { - $lock = false; - } - } - if (is_bool($lock)) { - Cache::set($cachekey, getmypid(), 300); - $got_lock = true; - } - - dba::unlock(); - - if (!$got_lock && ($timeout > 0)) { - usleep(rand(10000, 200000)); - } - } while (!$got_lock && ((time() - $start) < $timeout)); - - return $got_lock; - } - - /** - * @brief Removes a lock if it was set by us - * - * @param string $key Name of the lock - * - * @return mixed - */ - public function releaseLock($key) - { - $cachekey = get_app()->get_hostname() . ";lock:" . $key; - $lock = Cache::get($cachekey); - - if (!is_bool($lock)) { - if ((int)$lock == getmypid()) { - Cache::delete($cachekey); - } - } - - return; - } - - /** - * @brief Removes all lock that were set by us - * - * @return void - */ - public function releaseAll() - { - // We cannot delete all cache entries, but this doesn't matter with memcache - return; - } -} diff --git a/src/Util/Lock/SemaphoreLockDriver.php b/src/Util/Lock/SemaphoreLockDriver.php deleted file mode 100644 index 1cd3af1411..0000000000 --- a/src/Util/Lock/SemaphoreLockDriver.php +++ /dev/null @@ -1,82 +0,0 @@ -=')) { - self::$semaphore[$key] = sem_get(self::semaphoreKey($key)); - if (self::$semaphore[$key]) { - return sem_acquire(self::$semaphore[$key], ($timeout == 0)); - } - } - } - - /** - * @brief Removes a lock if it was set by us - * - * @param string $key Name of the lock - * - * @return mixed - */ - public function releaseLock($key) - { - if (function_exists('sem_get') && version_compare(PHP_VERSION, '5.6.1', '>=')) { - if (empty(self::$semaphore[$key])) { - return false; - } else { - $success = @sem_release(self::$semaphore[$key]); - unset(self::$semaphore[$key]); - return $success; - } - } - } - - /** - * @brief Removes all lock that were set by us - * - * @return void - */ - public function releaseAll() - { - // not needed/supported - return; - } -}