From: Mikael Nordfeldth Date: Tue, 17 Feb 2015 17:55:12 +0000 (+0100) Subject: urlhash first steps, including scripts/upgrade.php X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=5a8f24eb46a3f4c5209c496918066bdbdcb4ec82;p=quix0rs-gnu-social.git urlhash first steps, including scripts/upgrade.php --- diff --git a/classes/File.php b/classes/File.php index ce2f940161..dad54166b1 100644 --- a/classes/File.php +++ b/classes/File.php @@ -26,7 +26,8 @@ class File extends Managed_DataObject { public $__table = 'file'; // table name public $id; // int(4) primary_key not_null - public $url; // varchar(255) unique_key + public $urlhash; // varchar(64) unique_key + public $url; // text public $mimetype; // varchar(50) public $size; // int(4) public $title; // varchar(255) @@ -37,12 +38,15 @@ class File extends Managed_DataObject public $height; // int(4) public $modified; // timestamp() not_null default_CURRENT_TIMESTAMP + const URLHASH_ALG = 'sha256'; + public static function schemaDef() { return array( 'fields' => array( 'id' => array('type' => 'serial', 'not null' => true), - 'url' => array('type' => 'varchar', 'length' => 255, 'description' => 'destination URL after following redirections'), + 'urlhash' => array('type' => 'varchar', 'length' => 64, 'description' => 'sha256 of destination URL (url field)'), + 'url' => array('type' => 'text', 'description' => 'destination URL after following redirections'), 'mimetype' => array('type' => 'varchar', 'length' => 50, 'description' => 'mime type of resource'), 'size' => array('type' => 'int', 'description' => 'size of resource when available'), 'title' => array('type' => 'varchar', 'length' => 255, 'description' => 'title of resource when available'), @@ -56,7 +60,7 @@ class File extends Managed_DataObject ), 'primary key' => array('id'), 'unique keys' => array( - 'file_url_key' => array('url'), + 'file_urlhash_key' => array('urlhash'), ), ); } @@ -77,10 +81,11 @@ class File extends Managed_DataObject // I don't know why we have to keep doing this but I'm adding this last check to avoid // uniqueness bugs. - $file = File::getKV('url', $given_url); + $file = File::getKV('urlhash', self::hashurl($given_url)); if (!$file instanceof File) { $file = new File; + $file->urlhash = self::hashurl($given_url); $file->url = $given_url; if (!empty($redir_data['protected'])) $file->protected = $redir_data['protected']; if (!empty($redir_data['title'])) $file->title = $redir_data['title']; @@ -462,7 +467,7 @@ class File extends Managed_DataObject if (!empty($this->filename)) { // A locally stored file, so let's generate a URL for our instance. $url = self::url($this->filename); - if ($url != $this->url) { + if (self::urlhash($url) !== $this->urlhash) { // For indexing purposes, in case we do a lookup on the 'url' field. // also we're fixing possible changes from http to https, or paths $this->updateUrl($url); @@ -476,14 +481,15 @@ class File extends Managed_DataObject public function updateUrl($url) { - $file = File::getKV('url', $url); + $file = File::getKV('urlhash', self::hashurl($url)); if ($file instanceof File) { throw new ServerException('URL already exists in DB'); } - $sql = 'UPDATE %1$s SET url=%2$s WHERE url=%3$s;'; + $sql = 'UPDATE %1$s SET urlhash=%2$s, url=%3$s WHERE urlhash=%4$s;'; $result = $this->query(sprintf($sql, $this->__table, + $this->_quote((string)self::hashurl($url)), $this->_quote((string)$url), - $this->_quote((string)$this->url))); + $this->_quote((string)$this->urlhash))); if ($result === false) { common_log_db_error($this, 'UPDATE', __FILE__); throw new ServerException("Could not UPDATE {$this->__table}.url"); @@ -502,9 +508,9 @@ class File extends Managed_DataObject function blowCache($last=false) { - self::blow('file:notice-ids:%s', $this->url); + self::blow('file:notice-ids:%s', $this->urlhash); if ($last) { - self::blow('file:notice-ids:%s;last', $this->url); + self::blow('file:notice-ids:%s;last', $this->urlhash); } self::blow('file:notice-count:%d', $this->id); } @@ -582,4 +588,12 @@ class File extends Managed_DataObject return $title ?: null; } + + static public function hashurl($url) + { + if (empty($url)) { + throw new Exception('No URL provided to hash algorithm.'); + } + return hash(self::URLHASH_ALG, $url); + } } diff --git a/scripts/upgrade.php b/scripts/upgrade.php index c221a495af..eeddf5559f 100644 --- a/scripts/upgrade.php +++ b/scripts/upgrade.php @@ -76,6 +76,7 @@ function updateSchemaCore() $schema = Schema::get(); $schemaUpdater = new SchemaUpdater($schema); foreach (tableDefs() as $table => $def) { + preAlterFixes($schemaUpdater, $table); $schemaUpdater->register($table, $def); } $schemaUpdater->checkSchema(); @@ -83,6 +84,40 @@ function updateSchemaCore() printfnq("DONE.\n"); } +function preAlterFixes($schemaUpdater, $table) +{ + switch ($table) { + case 'file': + $schemadef = $schemaUpdater->schema->getTableDef($table); + if (isset($schemadef['fields']['urlhash'])) { + // We already have the urlhash field, so no need to migrate it. + break; + } + printfnq("\nFound old $table table, upgrading it to contain 'urlhash' field...\n"); + // We have to create a urlhash that is _not_ the primary key, + // transfer data and THEN run checkSchema + $schemadef['fields']['urlhash'] = array ( + 'type' => 'varchar', + 'length' => 64, + 'description' => 'sha256 of destination URL after following redirections', + ); + $schemaUpdater->schema->ensureTable($table, $schemadef); + printfnq("DONE.\n"); + + $filefix = new File(); + // urlhash is hash('sha256', $url) in the File table + printfnq("Updating urlhash fields in $table table...\n"); + $filefix->query(sprintf('UPDATE %1$s SET %2$s=%3$s;', + $schemaUpdater->schema->quoteIdentifier($table), + 'urlhash', + // The line below is "result of sha256 on column `url`" + 'SHA2(url, 256)')); + printfnq("DONE.\n"); + printfnq("Resuming core schema upgrade..."); + break; + } +} + function updateSchemaPlugins() { printfnq("Upgrading plugin schema...");