simgear/package/Install.cxx

   1 // Copyright (C) 2013  James Turner - zakalawe@mac.com
   2 //
   3 // This library is free software; you can redistribute it and/or
   4 // modify it under the terms of the GNU Library General Public
   5 // License as published by the Free Software Foundation; either
   6 // version 2 of the License, or (at your option) any later version.
   7 //
   8 // This library is distributed in the hope that it will be useful,
   9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11 // Library General Public License for more details.
  12 //
  13 // You should have received a copy of the GNU General Public License
  14 // along with this program; if not, write to the Free Software
  15 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  16 //
  17
  18 #include <simgear/package/Install.hxx>
  19
  20 #include <boost/foreach.hpp>
  21 #include <fstream>
  22
  23 #include <simgear/package/unzip.h>
  24 #include <simgear/package/md5.h>
  25 #include <simgear/package/untar.hxx>
  26
  27 #include <simgear/structure/exception.hxx>
  28 #include <simgear/props/props_io.hxx>
  29 #include <simgear/package/Catalog.hxx>
  30 #include <simgear/package/Package.hxx>
  31 #include <simgear/package/Root.hxx>
  32 #include <simgear/io/HTTPRequest.hxx>
  33 #include <simgear/io/HTTPClient.hxx>
  34 #include <simgear/misc/sg_dir.hxx>
  35 #include <simgear/misc/strutils.hxx>
  36
  37 extern "C" {
  38     void fill_memory_filefunc (zlib_filefunc_def*);
  39 }
  40
  41 namespace simgear {
  42
  43 namespace pkg {
  44
  45 class Install::PackageArchiveDownloader : public HTTP::Request
  46 {
  47 public:
  48     PackageArchiveDownloader(InstallRef aOwner) :
  49         HTTP::Request("" /* dummy URL */),
  50         m_owner(aOwner),
  51         m_downloaded(0)
  52     {
  53         m_urls = m_owner->package()->downloadUrls();
  54         if (m_urls.empty()) {
  55             throw sg_exception("no package download URLs");
  56         }
  57
  58         // TODO randomise order of m_urls
  59
  60         m_extractPath = aOwner->path().dir();
  61         m_extractPath.append("_extract_" + aOwner->package()->md5());
  62
  63         // clean up any existing files
  64         Dir d(m_extractPath);
  65         if (d.exists()) {
  66             d.remove(true /* recursive */);
  67         }
  68     }
  69
  70     ~PackageArchiveDownloader()
  71     {
  72         // always clean up our extraction dir: if we successfully downloaded
  73         // and installed it will be an empty dir, if we failed it might contain
  74         // (some) of the package files.
  75         Dir d(m_extractPath);
  76         if (d.exists()) {
  77             d.remove(true /* recursive */);
  78         }
  79     }
  80
  81     size_t downloadedBytes() const
  82     {
  83         return m_downloaded;
  84     }
  85
  86     int percentDownloaded() const
  87     {
  88         if (responseLength() <= 0) {
  89             return 0;
  90         }
  91
  92         return (m_downloaded * 100) / responseLength();
  93     }
  94 protected:
  95     virtual std::string url() const
  96     {
  97         return m_urls.front();
  98     }
  99
 100     virtual void responseHeadersComplete()
 101     {
 102         Request::responseHeadersComplete();
 103
 104         Dir d(m_extractPath);
 105         d.create(0755);
 106
 107         memset(&m_md5, 0, sizeof(SG_MD5_CTX));
 108         SG_MD5Init(&m_md5);
 109     }
 110
 111     virtual void gotBodyData(const char* s, int n)
 112     {
 113         m_buffer += std::string(s, n);
 114         SG_MD5Update(&m_md5, (unsigned char*) s, n);
 115
 116         m_downloaded = m_buffer.size();
 117         m_owner->installProgress(m_buffer.size(), responseLength());
 118     }
 119
 120     virtual void onDone()
 121     {
 122         if (responseCode() != 200) {
 123             SG_LOG(SG_GENERAL, SG_ALERT, "download failure:" << responseCode() <<
 124                    "\n\t" << url());
 125             Delegate::StatusCode code = Delegate::FAIL_DOWNLOAD;
 126             if (responseCode() == 404) {
 127                 code = Delegate::FAIL_NOT_FOUND;
 128             }
 129
 130             doFailure(code);
 131             return;
 132         }
 133
 134         unsigned char digest[MD5_DIGEST_LENGTH];
 135         SG_MD5Final(digest, &m_md5);
 136         std::string const hex_md5 =
 137           strutils::encodeHex(digest, MD5_DIGEST_LENGTH);
 138
 139         if (hex_md5 != m_owner->package()->md5()) {
 140             SG_LOG(SG_GENERAL, SG_ALERT, "md5 verification failed:\n"
 141                 << "\t" << hex_md5 << "\n\t"
 142                 << m_owner->package()->md5() << "\n\t"
 143                 << "downloading from:" << url());
 144             doFailure(Delegate::FAIL_CHECKSUM);
 145             return;
 146         }
 147
 148         if (!extract()) {
 149             SG_LOG(SG_GENERAL, SG_WARN, "archive extraction failed");
 150             doFailure(Delegate::FAIL_EXTRACT);
 151             return;
 152         }
 153
 154         if (m_owner->path().exists()) {
 155             Dir destDir(m_owner->path());
 156             destDir.remove(true /* recursive */);
 157         }
 158
 159         // build a path like /path/to/packages/org.some.catalog/Aircraft/extract_xxxx/MyAircraftDir
 160         SGPath extractedPath = m_extractPath;
 161         extractedPath.append(m_owner->package()->dirName());
 162
 163         // rename it to path/to/packages/org.some.catalog/Aircraft/MyAircraftDir
 164         bool ok = extractedPath.rename(m_owner->path());
 165         if (!ok) {
 166             doFailure(Delegate::FAIL_FILESYSTEM);
 167             return;
 168         }
 169
 170         // extract_xxxx directory is now empty, so remove it
 171         if (m_extractPath.exists()) {
 172             simgear::Dir(m_extractPath).remove();
 173         }
 174
 175         m_owner->m_revision = m_owner->package()->revision();
 176         m_owner->writeRevisionFile();
 177         m_owner->m_download.reset(); // so isDownloading reports false
 178
 179         m_owner->installResult(Delegate::STATUS_SUCCESS);
 180     }
 181
 182     virtual void onFail()
 183     {
 184         if (responseCode() == -1) {
 185             doFailure(Delegate::USER_CANCELLED);
 186         } else {
 187             doFailure(Delegate::FAIL_DOWNLOAD);
 188         }
 189     }
 190
 191 private:
 192
 193     void extractCurrentFile(unzFile zip, char* buffer, size_t bufferSize)
 194     {
 195         unz_file_info fileInfo;
 196         unzGetCurrentFileInfo(zip, &fileInfo,
 197             buffer, bufferSize,
 198             NULL, 0,  /* extra field */
 199             NULL, 0 /* comment field */);
 200
 201         std::string name(buffer);
 202     // no absolute paths, no 'up' traversals
 203     // we could also look for suspicious file extensions here (forbid .dll, .exe, .so)
 204         if ((name[0] == '/') || (name.find("../") != std::string::npos) || (name.find("..\\") != std::string::npos)) {
 205             throw sg_format_exception("Bad zip path", name);
 206         }
 207
 208         if (fileInfo.uncompressed_size == 0) {
 209             // assume it's a directory for now
 210             // since we create parent directories when extracting
 211             // a path, we're done here
 212             return;
 213         }
 214
 215         int result = unzOpenCurrentFile(zip);
 216         if (result != UNZ_OK) {
 217             throw sg_io_exception("opening current zip file failed", sg_location(name));
 218         }
 219
 220         std::ofstream outFile;
 221         bool eof = false;
 222         SGPath path(m_extractPath);
 223         path.append(name);
 224
 225     // create enclosing directory heirarchy as required
 226         Dir parentDir(path.dir());
 227         if (!parentDir.exists()) {
 228             bool ok = parentDir.create(0755);
 229             if (!ok) {
 230                 throw sg_io_exception("failed to create directory heirarchy for extraction", path.c_str());
 231             }
 232         }
 233
 234         outFile.open(path.c_str(), std::ios::binary | std::ios::trunc | std::ios::out);
 235         if (outFile.fail()) {
 236             throw sg_io_exception("failed to open output file for writing", path.c_str());
 237         }
 238
 239         while (!eof) {
 240             int bytes = unzReadCurrentFile(zip, buffer, bufferSize);
 241             if (bytes < 0) {
 242                 throw sg_io_exception("unzip failure reading curent archive", sg_location(name));
 243             } else if (bytes == 0) {
 244                 eof = true;
 245             } else {
 246                 outFile.write(buffer, bytes);
 247             }
 248         }
 249
 250         outFile.close();
 251         unzCloseCurrentFile(zip);
 252     }
 253
 254     bool extract()
 255     {
 256         const std::string u(url());
 257         const size_t ul(u.length());
 258         if (u.rfind(".zip") == (ul - 4)) {
 259             return extractUnzip();
 260         }
 261
 262         if (u.rfind(".tar.gz") == (ul - 7)) {
 263             return extractTar();
 264         }
 265
 266         SG_LOG(SG_IO, SG_WARN, "unsupported archive format:" << u);
 267         return false;
 268     }
 269
 270     bool extractUnzip()
 271     {
 272         bool result = true;
 273         zlib_filefunc_def memoryAccessFuncs;
 274         fill_memory_filefunc(&memoryAccessFuncs);
 275
 276         char bufferName[128];
 277         snprintf(bufferName, 128, "%p+%lx", m_buffer.data(), m_buffer.size());
 278         unzFile zip = unzOpen2(bufferName, &memoryAccessFuncs);
 279
 280         const size_t BUFFER_SIZE = 32 * 1024;
 281         void* buf = malloc(BUFFER_SIZE);
 282
 283         try {
 284             int result = unzGoToFirstFile(zip);
 285             if (result != UNZ_OK) {
 286                 throw sg_exception("failed to go to first file in archive");
 287             }
 288
 289             while (true) {
 290                 extractCurrentFile(zip, (char*) buf, BUFFER_SIZE);
 291                 result = unzGoToNextFile(zip);
 292                 if (result == UNZ_END_OF_LIST_OF_FILE) {
 293                     break;
 294                 } else if (result != UNZ_OK) {
 295                     throw sg_io_exception("failed to go to next file in the archive");
 296                 }
 297             }
 298         } catch (sg_exception& ) {
 299             result = false;
 300         }
 301
 302         free(buf);
 303         unzClose(zip);
 304         return result;
 305     }
 306
 307     bool extractTar()
 308     {
 309         TarExtractor tx(m_extractPath);
 310         tx.extractBytes(m_buffer.data(), m_buffer.size());
 311         return !tx.hasError() && tx.isAtEndOfArchive();
 312     }
 313
 314     void doFailure(Delegate::StatusCode aReason)
 315     {
 316         Dir dir(m_extractPath);
 317         if (dir.exists()) {
 318             dir.remove(true /* recursive */);
 319         }
 320
 321         // TODO - try other mirrors
 322         m_owner->m_download.reset(); // ensure we get cleaned up
 323         m_owner->installResult(aReason);
 324     }
 325
 326     InstallRef m_owner;
 327     string_list m_urls;
 328     SG_MD5_CTX m_md5;
 329     std::string m_buffer;
 330     SGPath m_extractPath;
 331     size_t m_downloaded;
 332 };
 333
 334 ////////////////////////////////////////////////////////////////////
 335 Install::Install(PackageRef aPkg, const SGPath& aPath) :
 336     m_package(aPkg),
 337     m_path(aPath),
 338     m_status(Delegate::STATUS_IN_PROGRESS)
 339 {
 340     parseRevision();
 341     m_package->catalog()->root()->registerInstall(this);
 342 }
 343
 344 Install::~Install()
 345 {
 346 }
 347
 348 InstallRef Install::createFromPath(const SGPath& aPath, CatalogRef aCat)
 349 {
 350     std::string path = aPath.file();
 351     PackageRef pkg = aCat->getPackageByPath(path);
 352     if (!pkg)
 353         throw sg_exception("no package with path:" + path);
 354
 355     return new Install(pkg, aPath);
 356 }
 357
 358 void Install::parseRevision()
 359 {
 360     SGPath revisionFile = m_path;
 361     revisionFile.append(".revision");
 362     if (!revisionFile.exists()) {
 363         m_revision = 0;
 364         return;
 365     }
 366
 367     std::ifstream f(revisionFile.c_str(), std::ios::in);
 368     f >> m_revision;
 369 }
 370
 371 void Install::writeRevisionFile()
 372 {
 373     SGPath revisionFile = m_path;
 374     revisionFile.append(".revision");
 375     std::ofstream f(revisionFile.c_str(), std::ios::out | std::ios::trunc);
 376     f << m_revision << std::endl;
 377 }
 378
 379 bool Install::hasUpdate() const
 380 {
 381     return m_package->revision() > m_revision;
 382 }
 383
 384 void Install::startUpdate()
 385 {
 386     if (m_download) {
 387         return; // already active
 388     }
 389
 390     m_download = new PackageArchiveDownloader(this);
 391     m_package->catalog()->root()->makeHTTPRequest(m_download);
 392     m_package->catalog()->root()->startInstall(this);
 393 }
 394
 395 bool Install::uninstall()
 396 {
 397     Dir d(m_path);
 398     if (!d.remove(true)) {
 399         SG_LOG(SG_GENERAL, SG_ALERT, "package uninstall failed: couldn't remove path " << m_path);
 400         return false;
 401     }
 402
 403     m_package->catalog()->root()->unregisterInstall(this);
 404     return true;
 405 }
 406
 407 bool Install::isDownloading() const
 408 {
 409     return (m_download.valid());
 410 }
 411
 412 bool Install::isQueued() const
 413 {
 414     return m_package->catalog()->root()->isInstallQueued(const_cast<Install*>(this));
 415 }
 416
 417 int Install::downloadedPercent() const
 418 {
 419     if (!m_download.valid()) {
 420         return -1;
 421     }
 422
 423     PackageArchiveDownloader* dl = static_cast<PackageArchiveDownloader*>(m_download.get());
 424     return dl->percentDownloaded();
 425 }
 426
 427 size_t Install::downloadedBytes() const
 428 {
 429     if (!m_download.valid()) {
 430         return -1;
 431     }
 432
 433     PackageArchiveDownloader* dl = static_cast<PackageArchiveDownloader*>(m_download.get());
 434     return dl->downloadedBytes();
 435
 436 }
 437
 438 void Install::cancelDownload()
 439 {
 440     if (m_download.valid()) {
 441         m_package->catalog()->root()->cancelHTTPRequest(m_download, "User cancelled download");
 442     }
 443
 444     if (m_revision == 0) {
 445         SG_LOG(SG_GENERAL, SG_INFO, "cancel install of package, will unregister");
 446         m_package->catalog()->root()->unregisterInstall(this);
 447     }
 448
 449     m_package->catalog()->root()->cancelDownload(this);
 450 }
 451
 452 struct PathAppender
 453 {
 454     PathAppender(const SGPath& p) : m_path(p) {}
 455
 456     SGPath operator()(const std::string& s) const
 457     {
 458         SGPath p(m_path);
 459         p.append(s);
 460         return p;
 461     }
 462
 463     SGPath m_path;
 464 };
 465
 466 PathList Install::thumbnailPaths() const
 467 {
 468     const string_list& thumbs(m_package->thumbnails());
 469     PathList result;
 470     if (thumbs.empty())
 471         return result;
 472
 473     std::transform(thumbs.begin(), thumbs.end(),
 474                    std::back_inserter(result),
 475                    PathAppender(m_path));
 476     return result;
 477 }
 478
 479 SGPath Install::primarySetPath() const
 480 {
 481     SGPath setPath(m_path);
 482     std::string ps(m_package->id());
 483     setPath.append(ps + "-set.xml");
 484     return setPath;
 485 }
 486
 487 //------------------------------------------------------------------------------
 488 Install* Install::done(const Callback& cb)
 489 {
 490   if( m_status == Delegate::STATUS_SUCCESS )
 491     cb(this);
 492   else
 493     _cb_done.push_back(cb);
 494
 495   return this;
 496 }
 497
 498 //------------------------------------------------------------------------------
 499 Install* Install::fail(const Callback& cb)
 500 {
 501   if(    m_status != Delegate::STATUS_SUCCESS
 502       && m_status != Delegate::STATUS_IN_PROGRESS )
 503     cb(this);
 504   else
 505     _cb_fail.push_back(cb);
 506
 507   return this;
 508 }
 509
 510 //------------------------------------------------------------------------------
 511 Install* Install::always(const Callback& cb)
 512 {
 513   if( m_status != Delegate::STATUS_IN_PROGRESS )
 514     cb(this);
 515   else
 516     _cb_always.push_back(cb);
 517
 518   return this;
 519 }
 520
 521 //------------------------------------------------------------------------------
 522 Install* Install::progress(const ProgressCallback& cb)
 523 {
 524   _cb_progress.push_back(cb);
 525   return this;
 526 }
 527
 528 //------------------------------------------------------------------------------
 529 void Install::installResult(Delegate::StatusCode aReason)
 530 {
 531     m_package->catalog()->root()->finishInstall(this, aReason);
 532     if (aReason == Delegate::STATUS_SUCCESS) {
 533         _cb_done(this);
 534     } else {
 535         _cb_fail(this);
 536     }
 537
 538     _cb_always(this);
 539 }
 540
 541 //------------------------------------------------------------------------------
 542 void Install::installProgress(unsigned int aBytes, unsigned int aTotal)
 543 {
 544   m_package->catalog()->root()->installProgress(this, aBytes, aTotal);
 545   _cb_progress(this, aBytes, aTotal);
 546 }
 547
 548
 549 } // of namespace pkg
 550
 551 } // of namespace simgear