From 60a0c51e2b6b41f4d9584bd9a68553e12c52a0a7 Mon Sep 17 00:00:00 2001 From: James Turner Date: Wed, 12 Aug 2015 14:22:59 -0700 Subject: [PATCH] Package support hacking - rename failure code to status code, and add more to handle cancellation. - change caching of active Installs from Catalog to Root, to clarify ownership - expose download status on Install - adjust Delegate signatures to pass more information --- simgear/package/Catalog.cxx | 85 +++++----- simgear/package/Catalog.hxx | 29 ++-- simgear/package/Delegate.hxx | 37 ++--- simgear/package/Install.cxx | 234 ++++++++++++++++++++-------- simgear/package/Install.hxx | 33 +++- simgear/package/Package.cxx | 27 +++- simgear/package/Package.hxx | 6 + simgear/package/Root.cxx | 291 ++++++++++++++++++++++++++++++----- simgear/package/Root.hxx | 35 ++++- simgear/package/pkgutil.cxx | 36 +++-- 10 files changed, 594 insertions(+), 219 deletions(-) diff --git a/simgear/package/Catalog.cxx b/simgear/package/Catalog.cxx index c12a53a0..5d407fe5 100644 --- a/simgear/package/Catalog.cxx +++ b/simgear/package/Catalog.cxx @@ -77,8 +77,8 @@ public: m_owner(aOwner) { // refreshing - m_owner->changeStatus(Delegate::FAIL_IN_PROGRESS); - + m_owner->changeStatus(Delegate::STATUS_IN_PROGRESS); + SG_LOG(SG_GENERAL, SG_WARN, "downloading " << aUrl); } protected: @@ -138,7 +138,7 @@ protected: time(&m_owner->m_retrievedTime); m_owner->writeTimestamp(); - m_owner->refreshComplete(Delegate::CATALOG_REFRESHED); + m_owner->refreshComplete(Delegate::STATUS_REFRESHED); } private: @@ -200,6 +200,8 @@ CatalogRef Catalog::createFromPath(Root* aRoot, const SGPath& aPath) return NULL; } + } else { + SG_LOG(SG_GENERAL, SG_INFO, "creating catalog from:" << aPath); } CatalogRef c = new Catalog(aRoot); @@ -233,7 +235,7 @@ bool Catalog::uninstall() } changeStatus(atLeastOneFailure ? Delegate::FAIL_FILESYSTEM - : Delegate::FAIL_SUCCESS); + : Delegate::STATUS_SUCCESS); return ok; } @@ -283,31 +285,25 @@ Catalog::installedPackages() const return r; } -InstallRef Catalog::installForPackage(PackageRef pkg) const -{ - PackageInstallDict::const_iterator it = m_installed.find(pkg); - if (it == m_installed.end()) { - // check if it exists on disk, create - - SGPath p(pkg->pathOnDisk()); - if (p.exists()) { - return Install::createFromPath(p, CatalogRef(const_cast(this))); - } - - return NULL; - } - - return it->second; -} - void Catalog::refresh() { Downloader* dl = new Downloader(this, url()); - // will iupdate status to IN_PROGRESS + // will update status to IN_PROGRESS m_root->makeHTTPRequest(dl); - m_root->catalogRefreshBegin(this); } +struct FindById +{ + FindById(const std::string &id) : m_id(id) {} + + bool operator()(const PackageRef& ref) const + { + return ref->id() == m_id; + } + + std::string m_id; +}; + void Catalog::parseProps(const SGPropertyNode* aProps) { // copy everything except package children? @@ -321,8 +317,14 @@ void Catalog::parseProps(const SGPropertyNode* aProps) for (int i = 0; i < nChildren; i++) { const SGPropertyNode* pkgProps = aProps->getChild(i); if (strcmp(pkgProps->getName(), "package") == 0) { - PackageRef p = getPackageById(pkgProps->getStringValue("id")); - if (p) { + // can't use getPackageById here becuase the variant dict isn't + // built yet. Instead we need to look at m_packages directly. + + PackageList::iterator pit = std::find_if(m_packages.begin(), m_packages.end(), + FindById(pkgProps->getStringValue("id"))); + PackageRef p; + if (pit != m_packages.end()) { + p = *pit; // existing package p->updateFromProps(pkgProps); orphans.erase(p); // not an orphan @@ -374,7 +376,7 @@ void Catalog::parseProps(const SGPropertyNode* aProps) } // parsed XML ok, mark status as valid - changeStatus(Delegate::FAIL_SUCCESS); + changeStatus(Delegate::STATUS_SUCCESS); } PackageRef Catalog::getPackageById(const std::string& aId) const @@ -440,6 +442,10 @@ bool Catalog::needsRefresh() const std::string Catalog::getLocalisedString(const SGPropertyNode* aRoot, const char* aName) const { + if (!aRoot) { + return std::string(); + } + if (aRoot->hasChild(m_root->getLocale())) { const SGPropertyNode* localeRoot = aRoot->getChild(m_root->getLocale().c_str()); if (localeRoot->hasChild(aName)) { @@ -450,37 +456,20 @@ std::string Catalog::getLocalisedString(const SGPropertyNode* aRoot, const char* return aRoot->getStringValue(aName); } -void Catalog::refreshComplete(Delegate::FailureCode aReason) +void Catalog::refreshComplete(Delegate::StatusCode aReason) { - m_root->catalogRefreshComplete(this, aReason); + m_root->catalogRefreshStatus(this, aReason); changeStatus(aReason); } -void Catalog::registerInstall(Install* ins) -{ - if (!ins || ins->package()->catalog() != this) { - return; - } - - m_installed[ins->package()] = ins; -} - -void Catalog::unregisterInstall(Install* ins) -{ - if (!ins || ins->package()->catalog() != this) { - return; - } - - m_installed.erase(ins->package()); -} - -void Catalog::changeStatus(Delegate::FailureCode newStatus) +void Catalog::changeStatus(Delegate::StatusCode newStatus) { if (m_status == newStatus) { return; } m_status = newStatus; + m_root->catalogRefreshStatus(this, newStatus); m_statusCallbacks(this); } @@ -489,7 +478,7 @@ void Catalog::addStatusCallback(const Callback& cb) m_statusCallbacks.push_back(cb); } -Delegate::FailureCode Catalog::status() const +Delegate::StatusCode Catalog::status() const { return m_status; } diff --git a/simgear/package/Catalog.hxx b/simgear/package/Catalog.hxx index 4a21dc9d..e21df6a1 100644 --- a/simgear/package/Catalog.hxx +++ b/simgear/package/Catalog.hxx @@ -70,7 +70,7 @@ public: * uninstall this catalog entirely, including all installed packages */ bool uninstall(); - + /** * perform a refresh of the catalog contents */ @@ -109,8 +109,6 @@ public: PackageRef getPackageById(const std::string& aId) const; - InstallRef installForPackage(PackageRef pkg) const; - /** * test if the catalog data was retrieved longer ago than the * maximum permitted age for this catalog. @@ -123,11 +121,11 @@ public: * access the raw property data in the catalog */ SGPropertyNode* properties() const; - - Delegate::FailureCode status() const; - + + Delegate::StatusCode status() const; + typedef boost::function Callback; - + void addStatusCallback(const Callback& cb); template @@ -141,38 +139,29 @@ private: class Downloader; friend class Downloader; - friend class Install; - void registerInstall(Install* ins); - void unregisterInstall(Install* ins); - void parseProps(const SGPropertyNode* aProps); - void refreshComplete(Delegate::FailureCode aReason); + void refreshComplete(Delegate::StatusCode aReason); void parseTimestamp(); void writeTimestamp(); std::string getLocalisedString(const SGPropertyNode* aRoot, const char* aName) const; - void changeStatus(Delegate::FailureCode newStatus); + void changeStatus(Delegate::StatusCode newStatus); Root* m_root; SGPropertyNode_ptr m_props; SGPath m_installRoot; std::string m_url; - Delegate::FailureCode m_status; - + Delegate::StatusCode m_status; + PackageList m_packages; time_t m_retrievedTime; typedef std::map PackageWeakMap; PackageWeakMap m_variantDict; - // important that this is a weak-ref to Installs, - // since it is only cleaned up in the Install destructor - typedef std::map PackageInstallDict; - PackageInstallDict m_installed; - function_list m_statusCallbacks; }; diff --git a/simgear/package/Delegate.hxx b/simgear/package/Delegate.hxx index 799823fe..61f28564 100644 --- a/simgear/package/Delegate.hxx +++ b/simgear/package/Delegate.hxx @@ -18,6 +18,8 @@ #ifndef SG_PACKAGE_DELEGATE_HXX #define SG_PACKAGE_DELEGATE_HXX +#include + namespace simgear { @@ -26,6 +28,9 @@ namespace pkg class Install; class Catalog; + +typedef SGSharedPtr CatalogRef; +typedef SGSharedPtr InstallRef; /** * package delegate is the mechanism to discover progress / completion / @@ -35,39 +40,35 @@ class Delegate { public: typedef enum { - FAIL_SUCCESS = 0, ///< not a failure :) + STATUS_SUCCESS = 0, FAIL_UNKNOWN = 1, - FAIL_IN_PROGRESS, ///< downloading/installation in progress (not a failure :P) + STATUS_IN_PROGRESS, ///< downloading/installation in progress FAIL_CHECKSUM, ///< package MD5 verificstion failed FAIL_DOWNLOAD, ///< network issue FAIL_EXTRACT, ///< package archive failed to extract cleanly FAIL_FILESYSTEM, ///< unknown filesystem error occurred FAIL_VERSION, ///< version check mismatch - CATALOG_REFRESHED - } FailureCode; + STATUS_REFRESHED, + USER_CANCELLED + } StatusCode; virtual ~Delegate() { } - /** - * invoked when all catalogs have finished refreshing - either successfully - * or with a failure. - */ - virtual void refreshComplete() = 0; /** - * emitted when a catalog fails to refresh, due to a network issue or - * some other failure. + * emitted when a catalog refesh completes, either success or failure + * If catalog is null, this means /all/ catalogs have been refreshed */ - virtual void failedRefresh(Catalog*, FailureCode aReason) = 0; - - virtual void startInstall(Install* aInstall) = 0; - virtual void installProgress(Install* aInstall, unsigned int aBytes, unsigned int aTotal) = 0; - virtual void finishInstall(Install* aInstall) = 0; + virtual void catalogRefreshed(CatalogRef, StatusCode aReason) = 0; - - virtual void failedInstall(Install* aInstall, FailureCode aReason) = 0; + virtual void startInstall(InstallRef aInstall) = 0; + virtual void installProgress(InstallRef aInstall, unsigned int aBytes, unsigned int aTotal) = 0; + virtual void finishInstall(InstallRef aInstall, StatusCode aReason) = 0; + virtual void dataForThumbnail(const std::string& aThumbnailUrl, + size_t lenth, const uint8_t* bytes) + {} }; } // of namespace pkg diff --git a/simgear/package/Install.cxx b/simgear/package/Install.cxx index 2ca01122..0217e457 100644 --- a/simgear/package/Install.cxx +++ b/simgear/package/Install.cxx @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -37,7 +38,7 @@ extern "C" { } namespace simgear { - + namespace pkg { class Install::PackageArchiveDownloader : public HTTP::Request @@ -45,43 +46,63 @@ class Install::PackageArchiveDownloader : public HTTP::Request public: PackageArchiveDownloader(InstallRef aOwner) : HTTP::Request("" /* dummy URL */), - m_owner(aOwner) + m_owner(aOwner), + m_downloaded(0) { m_urls = m_owner->package()->downloadUrls(); if (m_urls.empty()) { throw sg_exception("no package download URLs"); } - + // TODO randomise order of m_urls - + m_extractPath = aOwner->path().dir(); - m_extractPath.append("_DOWNLOAD"); // add some temporary value - + m_extractPath.append("_extract_" + aOwner->package()->md5()); + + // clean up any existing files + Dir d(m_extractPath); + if (d.exists()) { + d.remove(true /* recursive */); + } + } + + size_t downloadedBytes() const + { + return m_downloaded; + } + + int percentDownloaded() const + { + if (responseLength() <= 0) { + return 0; + } + + return (m_downloaded * 100) / responseLength(); } - protected: virtual std::string url() const { return m_urls.front(); } - + virtual void responseHeadersComplete() { Dir d(m_extractPath); - d.create(0755); - + d.create(0755); + memset(&m_md5, 0, sizeof(SG_MD5_CTX)); SG_MD5Init(&m_md5); } - + virtual void gotBodyData(const char* s, int n) { m_buffer += std::string(s, n); SG_MD5Update(&m_md5, (unsigned char*) s, n); - + + m_downloaded = m_buffer.size(); m_owner->installProgress(m_buffer.size(), responseLength()); } - + virtual void onDone() { if (responseCode() != 200) { @@ -103,65 +124,76 @@ protected: doFailure(Delegate::FAIL_CHECKSUM); return; } - + if (!extractUnzip()) { SG_LOG(SG_GENERAL, SG_WARN, "zip extraction failed"); doFailure(Delegate::FAIL_EXTRACT); return; } - + if (m_owner->path().exists()) { //std::cout << "removing existing path" << std::endl; Dir destDir(m_owner->path()); destDir.remove(true /* recursive */); } - + m_extractPath.append(m_owner->package()->id()); bool ok = m_extractPath.rename(m_owner->path()); if (!ok) { doFailure(Delegate::FAIL_FILESYSTEM); return; } - + m_owner->m_revision = m_owner->package()->revision(); m_owner->writeRevisionFile(); - m_owner->installResult(Delegate::FAIL_SUCCESS); + m_owner->m_download.reset(); // so isDownloading reports false + + m_owner->installResult(Delegate::STATUS_SUCCESS); + } + + virtual void onFail() + { + if (responseCode() == -1) { + doFailure(Delegate::USER_CANCELLED); + } else { + doFailure(Delegate::FAIL_DOWNLOAD); + } } - + private: void extractCurrentFile(unzFile zip, char* buffer, size_t bufferSize) { unz_file_info fileInfo; - unzGetCurrentFileInfo(zip, &fileInfo, - buffer, bufferSize, + unzGetCurrentFileInfo(zip, &fileInfo, + buffer, bufferSize, NULL, 0, /* extra field */ NULL, 0 /* comment field */); - + std::string name(buffer); // no absolute paths, no 'up' traversals // we could also look for suspicious file extensions here (forbid .dll, .exe, .so) if ((name[0] == '/') || (name.find("../") != std::string::npos) || (name.find("..\\") != std::string::npos)) { throw sg_format_exception("Bad zip path", name); } - + if (fileInfo.uncompressed_size == 0) { // assume it's a directory for now // since we create parent directories when extracting // a path, we're done here return; } - + int result = unzOpenCurrentFile(zip); if (result != UNZ_OK) { throw sg_io_exception("opening current zip file failed", sg_location(name)); } - + std::ofstream outFile; bool eof = false; SGPath path(m_extractPath); path.append(name); - + // create enclosing directory heirarchy as required Dir parentDir(path.dir()); if (!parentDir.exists()) { @@ -170,12 +202,12 @@ private: throw sg_io_exception("failed to create directory heirarchy for extraction", path.c_str()); } } - + outFile.open(path.c_str(), std::ios::binary | std::ios::trunc | std::ios::out); if (outFile.fail()) { throw sg_io_exception("failed to open output file for writing", path.c_str()); } - + while (!eof) { int bytes = unzReadCurrentFile(zip, buffer, bufferSize); if (bytes < 0) { @@ -186,30 +218,30 @@ private: outFile.write(buffer, bytes); } } - + outFile.close(); unzCloseCurrentFile(zip); } - + bool extractUnzip() { bool result = true; zlib_filefunc_def memoryAccessFuncs; fill_memory_filefunc(&memoryAccessFuncs); - + char bufferName[128]; snprintf(bufferName, 128, "%p+%lx", m_buffer.data(), m_buffer.size()); unzFile zip = unzOpen2(bufferName, &memoryAccessFuncs); - + const size_t BUFFER_SIZE = 32 * 1024; void* buf = malloc(BUFFER_SIZE); - + try { int result = unzGoToFirstFile(zip); if (result != UNZ_OK) { throw sg_exception("failed to go to first file in archive"); } - + while (true) { extractCurrentFile(zip, (char*) buf, BUFFER_SIZE); result = unzGoToNextFile(zip); @@ -222,48 +254,44 @@ private: } catch (sg_exception& e) { result = false; } - + free(buf); unzClose(zip); return result; } - - void doFailure(Delegate::FailureCode aReason) + + void doFailure(Delegate::StatusCode aReason) { Dir dir(m_extractPath); - dir.remove(true /* recursive */); - - if (m_urls.size() == 1) { - std::cout << "failure:" << aReason << std::endl; - m_owner->installResult(aReason); - return; + if (dir.exists()) { + dir.remove(true /* recursive */); } - - std::cout << "retrying download" << std::endl; - m_urls.erase(m_urls.begin()); // pop first URL + + // TODO - try other mirrors + m_owner->m_download.reset(); // ensure we get cleaned up + m_owner->installResult(aReason); } - + InstallRef m_owner; string_list m_urls; SG_MD5_CTX m_md5; std::string m_buffer; SGPath m_extractPath; + size_t m_downloaded; }; //////////////////////////////////////////////////////////////////// Install::Install(PackageRef aPkg, const SGPath& aPath) : m_package(aPkg), m_path(aPath), - m_download(NULL), - _status(Delegate::FAIL_IN_PROGRESS) + m_status(Delegate::STATUS_IN_PROGRESS) { parseRevision(); - m_package->catalog()->registerInstall(this); + m_package->catalog()->root()->registerInstall(this); } - + Install::~Install() { - m_package->catalog()->unregisterInstall(this); } InstallRef Install::createFromPath(const SGPath& aPath, CatalogRef aCat) @@ -272,7 +300,7 @@ InstallRef Install::createFromPath(const SGPath& aPath, CatalogRef aCat) PackageRef pkg = aCat->getPackageById(id); if (!pkg) throw sg_exception("no package with id:" + id); - + return new Install(pkg, aPath); } @@ -284,7 +312,7 @@ void Install::parseRevision() m_revision = 0; return; } - + std::ifstream f(revisionFile.c_str(), std::ios::in); f >> m_revision; } @@ -307,7 +335,7 @@ void Install::startUpdate() if (m_download) { return; // already active } - + m_download = new PackageArchiveDownloader(this); m_package->catalog()->root()->makeHTTPRequest(m_download); m_package->catalog()->root()->startInstall(this); @@ -320,20 +348,95 @@ bool Install::uninstall() SG_LOG(SG_GENERAL, SG_ALERT, "package uninstall failed: couldn't remove path " << m_path); return false; } - - m_package->catalog()->unregisterInstall(this); + + m_package->catalog()->root()->unregisterInstall(this); return true; } bool Install::isDownloading() const { - return (m_download != NULL); + return (m_download.valid()); +} + +bool Install::isQueued() const +{ + return m_package->catalog()->root()->isInstallQueued(const_cast(this)); +} + +int Install::downloadedPercent() const +{ + if (!m_download.valid()) { + return -1; + } + + PackageArchiveDownloader* dl = static_cast(m_download.get()); + return dl->percentDownloaded(); +} + +size_t Install::downloadedBytes() const +{ + if (!m_download.valid()) { + return -1; + } + + PackageArchiveDownloader* dl = static_cast(m_download.get()); + return dl->downloadedBytes(); + +} + +void Install::cancelDownload() +{ + if (m_download.valid()) { + m_download->abort("User cancelled download"); + } + + if (m_revision == 0) { + SG_LOG(SG_GENERAL, SG_INFO, "cancel install of package, will unregister"); + m_package->catalog()->root()->unregisterInstall(this); + } + + m_package->catalog()->root()->cancelDownload(this); +} + +struct PathAppender +{ + PathAppender(const SGPath& p) : m_path(p) {} + + SGPath operator()(const std::string& s) const + { + SGPath p(m_path); + p.append(s); + return p; + } + + SGPath m_path; +}; + +PathList Install::thumbnailPaths() const +{ + const string_list& thumbs(m_package->thumbnails()); + PathList result; + if (thumbs.empty()) + return result; + + std::transform(thumbs.begin(), thumbs.end(), + std::back_inserter(result), + PathAppender(m_path)); + return result; +} + +SGPath Install::primarySetPath() const +{ + SGPath setPath(m_path); + std::string ps(m_package->id()); + setPath.append(ps + "-set.xml"); + return setPath; } //------------------------------------------------------------------------------ Install* Install::done(const Callback& cb) { - if( _status == Delegate::FAIL_SUCCESS ) + if( m_status == Delegate::STATUS_SUCCESS ) cb(this); else _cb_done.push_back(cb); @@ -344,8 +447,8 @@ Install* Install::done(const Callback& cb) //------------------------------------------------------------------------------ Install* Install::fail(const Callback& cb) { - if( _status != Delegate::FAIL_SUCCESS - && _status != Delegate::FAIL_IN_PROGRESS ) + if( m_status != Delegate::STATUS_SUCCESS + && m_status != Delegate::STATUS_IN_PROGRESS ) cb(this); else _cb_fail.push_back(cb); @@ -356,7 +459,7 @@ Install* Install::fail(const Callback& cb) //------------------------------------------------------------------------------ Install* Install::always(const Callback& cb) { - if( _status != Delegate::FAIL_IN_PROGRESS ) + if( m_status != Delegate::STATUS_IN_PROGRESS ) cb(this); else _cb_always.push_back(cb); @@ -372,13 +475,12 @@ Install* Install::progress(const ProgressCallback& cb) } //------------------------------------------------------------------------------ -void Install::installResult(Delegate::FailureCode aReason) +void Install::installResult(Delegate::StatusCode aReason) { - if (aReason == Delegate::FAIL_SUCCESS) { - m_package->catalog()->root()->finishInstall(this); + m_package->catalog()->root()->finishInstall(this, aReason); + if (aReason == Delegate::STATUS_SUCCESS) { _cb_done(this); } else { - m_package->catalog()->root()->failedInstall(this, aReason); _cb_fail(this); } diff --git a/simgear/package/Install.hxx b/simgear/package/Install.hxx index ae90353d..4e198eba 100644 --- a/simgear/package/Install.hxx +++ b/simgear/package/Install.hxx @@ -21,11 +21,13 @@ #include #include +#include #include #include #include #include +#include #include @@ -78,6 +80,31 @@ public: bool isDownloading() const; + bool isQueued() const; + + int downloadedPercent() const; + + size_t downloadedBytes() const; + + /** + * full path to the primary -set.xml file for this install + */ + SGPath primarySetPath() const; + + /** + * if a download is in progress, cancel it. If this is the first install + * of the package (as opposed to an update), the install will be cleaned + * up once the last reference is gone. + */ + void cancelDownload(); + + /** + * return the thumbnails associated with this install, but as locations + * on the file system, not URLs. It is assumed the order of thumbnails + * is consistent with the URLs returned from Package::thumbnailUrls() + */ + PathList thumbnailPaths() const; + /** * Set the handler to be called when the installation successfully * completes. @@ -147,16 +174,16 @@ private: void parseRevision(); void writeRevisionFile(); - void installResult(Delegate::FailureCode aReason); + void installResult(Delegate::StatusCode aReason); void installProgress(unsigned int aBytes, unsigned int aTotal); PackageRef m_package; unsigned int m_revision; ///< revision on disk SGPath m_path; ///< installation point on disk - PackageArchiveDownloader* m_download; + HTTP::Request_ptr m_download; - Delegate::FailureCode _status; + Delegate::StatusCode m_status; function_list _cb_done, _cb_fail, diff --git a/simgear/package/Package.cxx b/simgear/package/Package.cxx index 59480486..5f76d746 100644 --- a/simgear/package/Package.cxx +++ b/simgear/package/Package.cxx @@ -139,7 +139,7 @@ InstallRef Package::install() InstallRef Package::existingInstall(const InstallCallback& cb) const { - InstallRef install = m_catalog->installForPackage(const_cast(this)); + InstallRef install = m_catalog->root()->existingInstallForPackage(const_cast(this)); if( cb ) { @@ -169,6 +169,10 @@ std::string Package::md5() const unsigned int Package::revision() const { + if (!m_props) { + return 0; + } + return m_props->getIntValue("revision"); } @@ -200,15 +204,36 @@ SGPropertyNode* Package::properties() const string_list Package::thumbnailUrls() const { string_list r; + if (!m_props) { + return r; + } + BOOST_FOREACH(SGPropertyNode* dl, m_props->getChildren("thumbnail")) { r.push_back(dl->getStringValue()); } return r; } +string_list Package::thumbnails() const +{ + string_list r; + if (!m_props) { + return r; + } + + BOOST_FOREACH(SGPropertyNode* dl, m_props->getChildren("thumbnail-path")) { + r.push_back(dl->getStringValue()); + } + return r; +} + string_list Package::downloadUrls() const { string_list r; + if (!m_props) { + return r; + } + BOOST_FOREACH(SGPropertyNode* dl, m_props->getChildren("url")) { r.push_back(dl->getStringValue()); } diff --git a/simgear/package/Package.hxx b/simgear/package/Package.hxx index b7389920..9293ca0d 100644 --- a/simgear/package/Package.hxx +++ b/simgear/package/Package.hxx @@ -122,6 +122,11 @@ public: string_list thumbnailUrls() const; + /** + * thumbnail file paths within the package on disk + */ + string_list thumbnails() const; + /** * Packages we depend upon. * If the dependency list cannot be satisifed for some reason, @@ -132,6 +137,7 @@ private: SGPath pathOnDisk() const; friend class Catalog; + friend class Root; Package(const SGPropertyNode* aProps, CatalogRef aCatalog); diff --git a/simgear/package/Root.cxx b/simgear/package/Root.cxx index a9127b1f..a0bef7c3 100644 --- a/simgear/package/Root.cxx +++ b/simgear/package/Root.cxx @@ -38,31 +38,152 @@ namespace simgear { namespace pkg { typedef std::map CatalogDict; +typedef std::vector DelegateVec; +typedef std::map MemThumbnailCache; +typedef std::deque StringDeque; + +class Root::ThumbnailDownloader : public HTTP::Request +{ +public: + ThumbnailDownloader(Root::RootPrivate* aOwner, const std::string& aUrl) : + HTTP::Request(aUrl), + m_owner(aOwner) + { + } + +protected: + virtual void gotBodyData(const char* s, int n) + { + m_buffer += std::string(s, n); + } + + virtual void onDone(); + +private: + Root::RootPrivate* m_owner; + std::string m_buffer; +}; class Root::RootPrivate { public: RootPrivate() : http(NULL), - maxAgeSeconds(60 * 60 * 24), - delegate(NULL) + maxAgeSeconds(60 * 60 * 24) { + } + + void fireStartInstall(InstallRef install) + { + DelegateVec::const_iterator it; + for (it = delegates.begin(); it != delegates.end(); ++it) { + (*it)->startInstall(install); + } + } + + void fireInstallProgress(InstallRef install, + unsigned int aBytes, unsigned int aTotal) + { + DelegateVec::const_iterator it; + for (it = delegates.begin(); it != delegates.end(); ++it) { + (*it)->installProgress(install, aBytes, aTotal); + } + } + + void fireFinishInstall(InstallRef install, Delegate::StatusCode status) + { + DelegateVec::const_iterator it; + for (it = delegates.begin(); it != delegates.end(); ++it) { + (*it)->finishInstall(install, status); + } + } + + void fireRefreshStatus(CatalogRef catalog, Delegate::StatusCode status) + { + DelegateVec::const_iterator it; + for (it = delegates.begin(); it != delegates.end(); ++it) { + (*it)->catalogRefreshed(catalog, status); + } + } + + + void thumbnailDownloadComplete(HTTP::Request_ptr request, + Delegate::StatusCode status, const std::string& bytes) + { + std::string u(request->url()); + SG_LOG(SG_IO, SG_INFO, "downloaded thumbnail:" << u); + if (status == Delegate::STATUS_SUCCESS) { + thumbnailCache[u] = bytes; + fireDataForThumbnail(u, bytes); + } + + downloadNextPendingThumbnail(); + } + + void fireDataForThumbnail(const std::string& aUrl, const std::string& bytes) + { + DelegateVec::const_iterator it; + const uint8_t* data = reinterpret_cast(bytes.data()); + for (it = delegates.begin(); it != delegates.end(); ++it) { + (*it)->dataForThumbnail(aUrl, bytes.size(), data); + } + } + + void downloadNextPendingThumbnail() + { + thumbnailDownloadRequest.clear(); + if (pendingThumbnails.empty()) { + return; + } + + std::string u = pendingThumbnails.front(); + pendingThumbnails.pop_front(); + thumbnailDownloadRequest = new Root::ThumbnailDownloader(this, u); + if (http) { + http->makeRequest(thumbnailDownloadRequest); + } else { + httpPendingRequests.push_back(thumbnailDownloadRequest); + } } + DelegateVec delegates; + SGPath path; std::string locale; HTTP::Client* http; CatalogDict catalogs; unsigned int maxAgeSeconds; - Delegate* delegate; std::string version; std::set refreshing; - std::deque updateDeque; + typedef std::deque UpdateDeque; + UpdateDeque updateDeque; std::deque httpPendingRequests; + + HTTP::Request_ptr thumbnailDownloadRequest; + StringDeque pendingThumbnails; + MemThumbnailCache thumbnailCache; + + typedef std::map InstallCache; + InstallCache m_installs; }; + +void Root::ThumbnailDownloader::onDone() +{ + if (responseCode() != 200) { + SG_LOG(SG_GENERAL, SG_ALERT, "thumbnail download failure:" << url()); + m_owner->thumbnailDownloadComplete(this, Delegate::FAIL_DOWNLOAD, std::string()); + return; + } + + m_owner->thumbnailDownloadComplete(this, Delegate::STATUS_SUCCESS, m_buffer); + //time(&m_owner->m_retrievedTime); + //m_owner->writeTimestamp(); + //m_owner->refreshComplete(Delegate::STATUS_REFRESHED); +} + SGPath Root::path() const { return d->path; @@ -180,6 +301,20 @@ CatalogList Root::catalogs() const return r; } +PackageList +Root::allPackages() const +{ + PackageList r; + + CatalogDict::const_iterator it = d->catalogs.begin(); + for (; it != d->catalogs.end(); ++it) { + const PackageList& r2(it->second->packages()); + r.insert(r.end(), r2.begin(), r2.end()); + } + + return r; +} + PackageList Root::packagesMatching(const SGPropertyNode* aFilter) const { @@ -210,17 +345,34 @@ Root::packagesNeedingUpdate() const void Root::refresh(bool aForce) { + bool didStartAny = false; CatalogDict::iterator it = d->catalogs.begin(); for (; it != d->catalogs.end(); ++it) { if (aForce || it->second->needsRefresh()) { it->second->refresh(); + didStartAny = true; } } + + if (!didStartAny) { + // signal refresh complete to the delegate already + d->fireRefreshStatus(CatalogRef(), Delegate::STATUS_REFRESHED); + } } -void Root::setDelegate(simgear::pkg::Delegate *aDelegate) +void Root::addDelegate(simgear::pkg::Delegate *aDelegate) { - d->delegate = aDelegate; + d->delegates.push_back(aDelegate); +} + +void Root::removeDelegate(simgear::pkg::Delegate *aDelegate) +{ + DelegateVec::iterator it = std::find(d->delegates.begin(), + d->delegates.end(), aDelegate); + if (it == d->delegates.end()) { + throw sg_exception("unknown delegate in removeDelegate"); + } + d->delegates.erase(it); } void Root::setLocale(const std::string& aLocale) @@ -254,18 +406,21 @@ void Root::scheduleToUpdate(InstallRef aInstall) } } +bool Root::isInstallQueued(InstallRef aInstall) const +{ + RootPrivate::UpdateDeque::const_iterator it = + std::find(d->updateDeque.begin(), d->updateDeque.end(), aInstall); + return (it != d->updateDeque.end()); +} + void Root::startInstall(InstallRef aInstall) { - if (d->delegate) { - d->delegate->startInstall(aInstall.ptr()); - } + d->fireStartInstall(aInstall); } void Root::installProgress(InstallRef aInstall, unsigned int aBytes, unsigned int aTotal) { - if (d->delegate) { - d->delegate->installProgress(aInstall.ptr(), aBytes, aTotal); - } + d->fireInstallProgress(aInstall, aBytes, aTotal); } void Root::startNext(InstallRef aCurrent) @@ -281,39 +436,49 @@ void Root::startNext(InstallRef aCurrent) } } -void Root::finishInstall(InstallRef aInstall) +void Root::finishInstall(InstallRef aInstall, Delegate::StatusCode aReason) { - if (d->delegate) { - d->delegate->finishInstall(aInstall.ptr()); + if (aReason != Delegate::STATUS_SUCCESS) { + SG_LOG(SG_GENERAL, SG_ALERT, "failed to install package:" + << aInstall->package()->id() << ":" << aReason); } + d->fireFinishInstall(aInstall, aReason); startNext(aInstall); } - -void Root::failedInstall(InstallRef aInstall, Delegate::FailureCode aReason) -{ - SG_LOG(SG_GENERAL, SG_ALERT, "failed to install package:" - << aInstall->package()->id() << ":" << aReason); - if (d->delegate) { - d->delegate->failedInstall(aInstall.ptr(), aReason); - } - startNext(aInstall); -} - -void Root::catalogRefreshBegin(CatalogRef aCat) +void Root::cancelDownload(InstallRef aInstall) { - d->refreshing.insert(aCat); + RootPrivate::UpdateDeque::iterator it = + std::find(d->updateDeque.begin(), d->updateDeque.end(), aInstall); + if (it != d->updateDeque.end()) { + bool startNext = (aInstall == d->updateDeque.front()); + d->updateDeque.erase(it); + if (startNext) { + if (!d->updateDeque.empty()) { + d->updateDeque.front()->startUpdate(); + } + } // of install was front item + } // of found install in queue } -void Root::catalogRefreshComplete(CatalogRef aCat, Delegate::FailureCode aReason) +void Root::catalogRefreshStatus(CatalogRef aCat, Delegate::StatusCode aReason) { CatalogDict::iterator catIt = d->catalogs.find(aCat->id()); - if (aReason != Delegate::FAIL_SUCCESS) { - if (d->delegate) { - d->delegate->failedRefresh(aCat, aReason); + d->fireRefreshStatus(aCat, aReason); + + if (aReason == Delegate::STATUS_IN_PROGRESS) { + d->refreshing.insert(aCat); + + if (catIt == d->catalogs.end()) { + // first fresh, add to our storage now + d->catalogs.insert(catIt, CatalogDict::value_type(aCat->id(), aCat)); } - + } else { + d->refreshing.erase(aCat); + } + + if ((aReason != Delegate::STATUS_REFRESHED) && (aReason != Delegate::STATUS_IN_PROGRESS)) { // if the failure is permanent, delete the catalog from our // list (don't touch it on disk) bool isPermanentFailure = (aReason == Delegate::FAIL_VERSION); @@ -323,16 +488,10 @@ void Root::catalogRefreshComplete(CatalogRef aCat, Delegate::FailureCode aReason d->catalogs.erase(catIt); } } - } else if (catIt == d->catalogs.end()) { - // first fresh, add to our storage now - d->catalogs.insert(catIt, CatalogDict::value_type(aCat->id(), aCat)); } - d->refreshing.erase(aCat); if (d->refreshing.empty()) { - if (d->delegate) { - d->delegate->refreshComplete(); - } + d->fireRefreshStatus(CatalogRef(), Delegate::STATUS_REFRESHED); } } @@ -357,7 +516,59 @@ bool Root::removeCatalogById(const std::string& aId) return ok; } + +void Root::requestThumbnailData(const std::string& aUrl) +{ + MemThumbnailCache::iterator it = d->thumbnailCache.find(aUrl); + if (it == d->thumbnailCache.end()) { + // insert into cache to mark as pending + d->pendingThumbnails.push_front(aUrl); + d->thumbnailCache[aUrl] = std::string(); + d->downloadNextPendingThumbnail(); + } else if (!it->second.empty()) { + // already loaded, fire data synchronously + d->fireDataForThumbnail(aUrl, it->second); + } else { + // in cache but empty data, still fetching + } +} + +InstallRef Root::existingInstallForPackage(PackageRef p) const +{ + RootPrivate::InstallCache::const_iterator it = + d->m_installs.find(p); + if (it == d->m_installs.end()) { + // check if it exists on disk, create + SGPath path(p->pathOnDisk()); + if (path.exists()) { + // this will add to our cache, and hence, modify m_installs + return Install::createFromPath(path, p->catalog()); + } + + return InstallRef(); + } + + return it->second; +} + +void Root::registerInstall(InstallRef ins) +{ + if (!ins.valid()) { + return; + } + + d->m_installs[ins->package()] = ins; +} +void Root::unregisterInstall(InstallRef ins) +{ + if (!ins .valid()) { + return; + } + + d->m_installs.erase(ins->package()); +} + } // of namespace pkg } // of namespace simgear diff --git a/simgear/package/Root.hxx b/simgear/package/Root.hxx index 2a06a6b1..11e9a647 100644 --- a/simgear/package/Root.hxx +++ b/simgear/package/Root.hxx @@ -62,8 +62,10 @@ public: void setLocale(const std::string& aLocale); - void setDelegate(Delegate* aDelegate); - + void addDelegate(Delegate* aDelegate); + + void removeDelegate(Delegate* aDelegate); + std::string getLocale() const; CatalogList catalogs() const; @@ -92,6 +94,11 @@ public: */ void refresh(bool aForce = false); + /** + * + */ + PackageList allPackages() const; + /** * retrieve packages matching a filter. * filter consists of required / minimum values, AND-ed together. @@ -115,21 +122,33 @@ public: * from the catalog too. */ bool removeCatalogById(const std::string& aId); + + /** + * request thumbnail data from the cache / network + */ + void requestThumbnailData(const std::string& aUrl); + + bool isInstallQueued(InstallRef aInstall) const; private: friend class Install; friend class Catalog; + friend class Package; - - void catalogRefreshBegin(CatalogRef aCat); - void catalogRefreshComplete(CatalogRef aCat, Delegate::FailureCode aReason); + InstallRef existingInstallForPackage(PackageRef p) const; + + void catalogRefreshStatus(CatalogRef aCat, Delegate::StatusCode aReason); void startNext(InstallRef aCurrent); void startInstall(InstallRef aInstall); void installProgress(InstallRef aInstall, unsigned int aBytes, unsigned int aTotal); - void finishInstall(InstallRef aInstall); - void failedInstall(InstallRef aInstall, Delegate::FailureCode aReason); - + void finishInstall(InstallRef aInstall, Delegate::StatusCode aReason); + void cancelDownload(InstallRef aInstall); + + void registerInstall(InstallRef ins); + void unregisterInstall(InstallRef ins); + + class ThumbnailDownloader; class RootPrivate; std::auto_ptr d; }; diff --git a/simgear/package/pkgutil.cxx b/simgear/package/pkgutil.cxx index 71fb2c6a..bcbe8318 100644 --- a/simgear/package/pkgutil.cxx +++ b/simgear/package/pkgutil.cxx @@ -34,22 +34,28 @@ bool keepRunning = true; class MyDelegate : public pkg::Delegate { public: - virtual void refreshComplete() + virtual void catalogRefreshed(pkg::CatalogRef aCatalog, StatusCode aReason) { + if (aReason == STATUS_REFRESHED) { + if (aCatalog.ptr() == NULL) { + cout << "refreshed all catalogs" << endl; + } else { + cout << "refreshed catalog " << aCatalog->url() << endl; + } + } else if (aReason == STATUS_IN_PROGRESS) { + cout << "started refresh of " << aCatalog->url() << endl; + } else { + cerr << "failed refresh of " << aCatalog->url() << ":" << aReason << endl; + } } - virtual void failedRefresh(pkg::Catalog* aCatalog, FailureCode aReason) - { - cerr << "failed refresh of " << aCatalog->description() << ":" << aReason << endl; - } - - virtual void startInstall(pkg::Install* aInstall) + virtual void startInstall(pkg::InstallRef aInstall) { _lastPercent = 999; cout << "starting install of " << aInstall->package()->name() << endl; } - virtual void installProgress(pkg::Install* aInstall, unsigned int bytes, unsigned int total) + virtual void installProgress(pkg::InstallRef aInstall, unsigned int bytes, unsigned int total) { unsigned int percent = (bytes * 100) / total; if (percent == _lastPercent) { @@ -60,15 +66,15 @@ public: cout << percent << "%" << endl; } - virtual void finishInstall(pkg::Install* aInstall) + virtual void finishInstall(pkg::InstallRef aInstall, StatusCode aReason) { - cout << "done install of " << aInstall->package()->name() << endl; + if (aReason == STATUS_SUCCESS) { + cout << "done install of " << aInstall->package()->name() << endl; + } else { + cerr << "failed install of " << aInstall->package()->name() << endl; + } } - virtual void failedInstall(pkg::Install* aInstall, FailureCode aReason) - { - cerr << "failed install of " << aInstall->package()->name() << endl; - } private: unsigned int _lastPercent; @@ -119,7 +125,7 @@ int main(int argc, char** argv) pkg::Root* root = new pkg::Root(Dir::current().path(), ""); MyDelegate dlg; - root->setDelegate(&dlg); + root->addDelegate(&dlg); cout << "Package root is:" << Dir::current().path() << endl; cout << "have " << root->catalogs().size() << " catalog(s)" << endl; -- 2.39.5