simgear/io/HTTPRepository.cxx

   1 // HTTPRepository.cxx -- plain HTTP TerraSync remote client
   2 //
   3 // Copyright (C) 20126  James Turner <zakalawe@mac.com>
   4 //
   5 // This program is free software; you can redistribute it and/or
   6 // modify it under the terms of the GNU General Public License as
   7 // published by the Free Software Foundation; either version 2 of the
   8 // License, or (at your option) any later version.
   9 //
  10 // This program is distributed in the hope that it will be useful, but
  11 // WITHOUT ANY WARRANTY; without even the implied warranty of
  12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 // General Public License for more details.
  14 //
  15 // You should have received a copy of the GNU General Public License
  16 // along with this program; if not, write to the Free Software
  17 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  18
  19 #include "HTTPRepository.hxx"
  20
  21 #include <iostream>
  22 #include <cstring>
  23 #include <cassert>
  24 #include <algorithm>
  25 #include <sstream>
  26 #include <map>
  27 #include <set>
  28 #include <fstream>
  29 #include <limits>
  30 #include <cstdlib>
  31
  32 #include <fcntl.h>
  33
  34 #include "simgear/debug/logstream.hxx"
  35 #include "simgear/misc/strutils.hxx"
  36 #include <simgear/misc/sg_dir.hxx>
  37 #include <simgear/io/HTTPClient.hxx>
  38 #include <simgear/io/sg_file.hxx>
  39 #include <simgear/misc/sgstream.hxx>
  40 #include <simgear/structure/exception.hxx>
  41
  42 #include <simgear/misc/sg_hash.hxx>
  43
  44 namespace simgear
  45 {
  46
  47     class HTTPDirectory;
  48
  49 class HTTPRepoPrivate
  50 {
  51 public:
  52     struct HashCacheEntry
  53     {
  54         std::string filePath;
  55         time_t modTime;
  56         size_t lengthBytes;
  57         std::string hashHex;
  58
  59     };
  60
  61     typedef std::vector<HashCacheEntry> HashCache;
  62     HashCache hashes;
  63
  64     HTTPRepoPrivate(HTTPRepository* parent) :
  65     p(parent),
  66     isUpdating(false),
  67     status(AbstractRepository::REPO_NO_ERROR)
  68     { ; }
  69
  70     HTTPRepository* p; // link back to outer
  71     HTTP::Client* http;
  72     std::string baseUrl;
  73     SGPath basePath;
  74     bool isUpdating;
  75     AbstractRepository::ResultCode status;
  76     HTTPDirectory* rootDir;
  77
  78     HTTP::Request_ptr updateFile(HTTPDirectory* dir, const std::string& name);
  79     HTTP::Request_ptr updateDir(HTTPDirectory* dir);
  80
  81     std::string hashForPath(const SGPath& p);
  82     void updatedFileContents(const SGPath& p, const std::string& newHash);
  83     void parseHashCache();
  84     std::string computeHashForPath(const SGPath& p);
  85     void writeHashCache();
  86
  87     void failedToGetRootIndex();
  88
  89     typedef std::vector<HTTP::Request_ptr> RequestVector;
  90     RequestVector requests;
  91
  92     void finishedRequest(const HTTP::Request_ptr& req);
  93
  94     HTTPDirectory* getOrCreateDirectory(const std::string& path);
  95     bool deleteDirectory(const std::string& path);
  96
  97     typedef std::vector<HTTPDirectory*> DirectoryVector;
  98     DirectoryVector directories;
  99
 100 };
 101
 102 class HTTPDirectory
 103 {
 104     struct ChildInfo
 105     {
 106         enum Type
 107         {
 108             FileType,
 109             DirectoryType
 110         };
 111
 112         ChildInfo(Type ty, const char* nameData, const char* hashData) :
 113             type(ty),
 114             name(nameData),
 115             hash(hashData ? hashData : ""),
 116             sizeInBytes(0)
 117         {
 118         }
 119
 120         ChildInfo(const ChildInfo& other) :
 121             type(other.type),
 122             name(other.name),
 123             hash(other.hash),
 124             sizeInBytes(other.sizeInBytes)
 125         { }
 126
 127         void setSize(const char* sizeData)
 128         {
 129             sizeInBytes = ::strtol(sizeData, NULL, 10);
 130         }
 131
 132         bool operator<(const ChildInfo& other) const
 133         {
 134             return name < other.name;
 135         }
 136
 137         Type type;
 138         std::string name, hash;
 139         size_t sizeInBytes;
 140     };
 141
 142     typedef std::vector<ChildInfo> ChildInfoList;
 143     ChildInfoList children;
 144
 145 public:
 146     HTTPDirectory(HTTPRepoPrivate* repo, const std::string& path) :
 147         _repository(repo),
 148         _relativePath(path)
 149   {
 150       SGPath p(absolutePath());
 151       if (p.exists()) {
 152 // what is indexValid for?
 153 //          bool indexValid = false;
 154           try {
 155               // already exists on disk
 156               parseDirIndex(children);
 157 //              indexValid = true;
 158               std::sort(children.begin(), children.end());
 159           } catch (sg_exception& e) {
 160               // parsing cache failed
 161               children.clear();
 162           }
 163       }
 164   }
 165
 166     HTTPRepoPrivate* repository() const
 167     {
 168         return _repository;
 169     }
 170
 171     std::string url() const
 172     {
 173         if (_relativePath.str().empty()) {
 174             return _repository->baseUrl;
 175         }
 176
 177         return _repository->baseUrl + "/" + _relativePath.str();
 178     }
 179
 180     void dirIndexUpdated(const std::string& hash)
 181     {
 182         SGPath fpath(_relativePath);
 183         fpath.append(".dirindex");
 184         _repository->updatedFileContents(fpath, hash);
 185
 186         children.clear();
 187         parseDirIndex(children);
 188         std::sort(children.begin(), children.end());
 189     }
 190
 191     void failedToUpdate()
 192     {
 193         if (_relativePath.isNull()) {
 194             // root dir failed
 195             _repository->failedToGetRootIndex();
 196         } else {
 197             SG_LOG(SG_TERRASYNC, SG_WARN, "failed to update dir:" << _relativePath);
 198         }
 199     }
 200
 201     void updateChildrenBasedOnHash()
 202     {
 203         SG_LOG(SG_TERRASYNC, SG_DEBUG, "updated children for:" << relativePath());
 204
 205         string_list indexNames = indexChildren(),
 206             toBeUpdated, orphans;
 207         simgear::Dir d(absolutePath());
 208         PathList fsChildren = d.children(0);
 209         PathList::const_iterator it = fsChildren.begin();
 210
 211         for (; it != fsChildren.end(); ++it) {
 212             ChildInfo info(it->isDir() ? ChildInfo::DirectoryType : ChildInfo::FileType,
 213                            it->file().c_str(), NULL);
 214             std::string hash = hashForChild(info);
 215
 216             ChildInfoList::iterator c = findIndexChild(it->file());
 217             if (c == children.end()) {
 218                 orphans.push_back(it->file());
 219             } else if (c->hash != hash) {
 220                 // file exists, but hash mismatch, schedule update
 221                 if (!hash.empty()) {
 222                     SG_LOG(SG_TERRASYNC, SG_INFO, "file exists but hash is wrong for:" << c->name);
 223                 }
 224
 225                 toBeUpdated.push_back(c->name);
 226             } else {
 227                 // file exists and hash is valid. If it's a directory,
 228                 // perform a recursive check.
 229                 if (c->type == ChildInfo::DirectoryType) {
 230                     SGPath p(relativePath());
 231                     p.append(c->name);
 232                     HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str());
 233                     childDir->updateChildrenBasedOnHash();
 234                 } else {
 235                     SG_LOG(SG_TERRASYNC, SG_INFO, "existing file is ok:" << c->name);
 236                 }
 237             }
 238
 239             // remove existing file system children from the index list,
 240             // so we can detect new children
 241             string_list::iterator it = std::find(indexNames.begin(), indexNames.end(), c->name);
 242             if (it != indexNames.end()) {
 243                 indexNames.erase(it);
 244             }
 245         } // of real children iteration
 246
 247         // all remaining names in indexChilden are new children
 248         toBeUpdated.insert(toBeUpdated.end(), indexNames.begin(), indexNames.end());
 249
 250         removeOrphans(orphans);
 251         scheduleUpdates(toBeUpdated);
 252     }
 253
 254     void removeOrphans(const string_list& orphans)
 255     {
 256         string_list::const_iterator it;
 257         for (it = orphans.begin(); it != orphans.end(); ++it) {
 258             removeChild(*it);
 259         }
 260     }
 261
 262     string_list indexChildren() const
 263     {
 264         string_list r;
 265         r.reserve(children.size());
 266         ChildInfoList::const_iterator it;
 267         for (it=children.begin(); it != children.end(); ++it) {
 268             r.push_back(it->name);
 269         }
 270         return r;
 271     }
 272
 273     void scheduleUpdates(const string_list& names)
 274     {
 275         string_list::const_iterator it;
 276         for (it = names.begin(); it != names.end(); ++it) {
 277             ChildInfoList::iterator cit = findIndexChild(*it);
 278             if (cit == children.end()) {
 279                 SG_LOG(SG_TERRASYNC, SG_WARN, "scheduleUpdate, unknown child:" << *it);
 280                 continue;
 281             }
 282
 283             if (cit->type == ChildInfo::FileType) {
 284                 _repository->updateFile(this, *it);
 285             } else {
 286                 SGPath p(relativePath());
 287                 p.append(*it);
 288                 HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str());
 289                 _repository->updateDir(childDir);
 290             }
 291         }
 292     }
 293
 294     SGPath absolutePath() const
 295     {
 296         SGPath r(_repository->basePath);
 297         r.append(_relativePath.str());
 298         return r;
 299     }
 300
 301     SGPath relativePath() const
 302     {
 303         return _relativePath;
 304     }
 305
 306     void didUpdateFile(const std::string& file, const std::string& hash)
 307     {
 308         SGPath fpath(_relativePath);
 309         fpath.append(file);
 310         _repository->updatedFileContents(fpath, hash);
 311         SG_LOG(SG_TERRASYNC, SG_INFO, "did update:" << fpath);
 312     }
 313
 314     void didFailToUpdateFile(const std::string& file)
 315     {
 316         SGPath fpath(_relativePath);
 317         fpath.append(file);
 318         SG_LOG(SG_TERRASYNC, SG_WARN, "failed to update:" << fpath);
 319     }
 320 private:
 321
 322     struct ChildWithName
 323     {
 324         ChildWithName(const std::string& n) : name(n) {}
 325         std::string name;
 326
 327         bool operator()(const ChildInfo& info) const
 328         { return info.name == name; }
 329     };
 330
 331     ChildInfoList::iterator findIndexChild(const std::string& name)
 332     {
 333         return std::find_if(children.begin(), children.end(), ChildWithName(name));
 334     }
 335
 336     void parseDirIndex(ChildInfoList& children)
 337     {
 338         SGPath p(absolutePath());
 339         p.append(".dirindex");
 340         std::ifstream indexStream( p.str().c_str(), std::ios::in );
 341
 342         if ( !indexStream.is_open() ) {
 343             throw sg_io_exception("cannot open dirIndex file", p);
 344         }
 345
 346         char lineBuffer[512];
 347         char* lastToken;
 348
 349         while (!indexStream.eof() ) {
 350             indexStream.getline(lineBuffer, 512);
 351             lastToken = 0;
 352             char* typeData = ::strtok_r(lineBuffer, ":", &lastToken);
 353             if (!typeData) {
 354                 continue; // skip blank line
 355             }
 356
 357             if (!typeData) {
 358                 // malformed entry
 359                 throw sg_io_exception("Malformed dir index file", p);
 360             }
 361
 362             if (!strcmp(typeData, "version")) {
 363                 continue;
 364             } else if (!strcmp(typeData, "path")) {
 365                 continue;
 366             }
 367
 368             char* nameData = ::strtok_r(NULL, ":", &lastToken);
 369             char* hashData = ::strtok_r(NULL, ":", &lastToken);
 370             char* sizeData = ::strtok_r(NULL, ":", &lastToken);
 371
 372             if (typeData[0] == 'f') {
 373                 children.push_back(ChildInfo(ChildInfo::FileType, nameData, hashData));
 374             } else if (typeData[0] == 'd') {
 375                 children.push_back(ChildInfo(ChildInfo::DirectoryType, nameData, hashData));
 376             } else {
 377                 throw sg_io_exception("Malformed line code in dir index file", p);
 378             }
 379
 380             if (sizeData) {
 381                 children.back().setSize(sizeData);
 382             }
 383         }
 384     }
 385
 386     void removeChild(const std::string& name)
 387     {
 388         SGPath p(absolutePath());
 389         p.append(name);
 390         bool ok;
 391
 392         SGPath fpath(_relativePath);
 393         fpath.append(name);
 394
 395         if (p.isDir()) {
 396             ok = _repository->deleteDirectory(fpath.str());
 397         } else {
 398             // remove the hash cache entry
 399             _repository->updatedFileContents(fpath, std::string());
 400             ok = p.remove();
 401         }
 402
 403         if (!ok) {
 404             SG_LOG(SG_TERRASYNC, SG_WARN, "removal failed for:" << p);
 405         }
 406     }
 407
 408     std::string hashForChild(const ChildInfo& child) const
 409     {
 410         SGPath p(absolutePath());
 411         p.append(child.name);
 412         if (child.type == ChildInfo::DirectoryType) {
 413             p.append(".dirindex");
 414         }
 415         return _repository->hashForPath(p);
 416     }
 417
 418   HTTPRepoPrivate* _repository;
 419   SGPath _relativePath; // in URL and file-system space
 420
 421
 422 };
 423
 424 HTTPRepository::HTTPRepository(const SGPath& base, HTTP::Client *cl) :
 425     _d(new HTTPRepoPrivate(this))
 426 {
 427     _d->http = cl;
 428     _d->basePath = base;
 429     _d->rootDir = new HTTPDirectory(_d.get(), "");
 430 }
 431
 432 HTTPRepository::~HTTPRepository()
 433 {
 434 }
 435
 436 void HTTPRepository::setBaseUrl(const std::string &url)
 437 {
 438   _d->baseUrl = url;
 439 }
 440
 441 std::string HTTPRepository::baseUrl() const
 442 {
 443   return _d->baseUrl;
 444 }
 445
 446 HTTP::Client* HTTPRepository::http() const
 447 {
 448   return _d->http;
 449 }
 450
 451 SGPath HTTPRepository::fsBase() const
 452 {
 453   return SGPath();
 454 }
 455
 456 void HTTPRepository::update()
 457 {
 458     if (_d->isUpdating) {
 459         return;
 460     }
 461
 462     _d->status = REPO_NO_ERROR;
 463     _d->isUpdating = true;
 464     _d->updateDir(_d->rootDir);
 465 }
 466
 467 bool HTTPRepository::isDoingSync() const
 468 {
 469     if (_d->status != REPO_NO_ERROR) {
 470         return false;
 471     }
 472
 473     return _d->isUpdating;
 474 }
 475
 476 AbstractRepository::ResultCode
 477 HTTPRepository::failure() const
 478 {
 479     return _d->status;
 480 }
 481
 482     class FileGetRequest : public HTTP::Request
 483     {
 484     public:
 485         FileGetRequest(HTTPDirectory* d, const std::string& file) :
 486             HTTP::Request(makeUrl(d, file)),
 487             directory(d),
 488             fileName(file),
 489             fd(-1)
 490         {
 491             SG_LOG(SG_TERRASYNC, SG_INFO, "will GET file " << url());
 492
 493         }
 494
 495     protected:
 496         virtual void gotBodyData(const char* s, int n)
 497         {
 498             if (fd < 0) {
 499                 SGPath p(pathInRepo());
 500 #ifdef SG_WINDOWS
 501                 int mode = 00666;
 502 #else
 503                 mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
 504 #endif
 505                 fd = ::open(p.c_str(), O_CREAT | O_TRUNC | O_RDWR, mode);
 506                 if (fd < 0) {
 507                     SG_LOG(SG_TERRASYNC, SG_WARN, "unable to create file " << p);
 508                     // fail
 509                 }
 510                 sha1_init(&hashContext);
 511             }
 512
 513             ::write(fd, s, n);
 514             sha1_write(&hashContext, s, n);
 515
 516         }
 517
 518         virtual void onDone()
 519         {
 520             ::close(fd);
 521             if (responseCode() == 200) {
 522                 std::string hash = strutils::encodeHex((char*) sha1_result(&hashContext));
 523                 directory->didUpdateFile(fileName, hash);
 524
 525                 SG_LOG(SG_TERRASYNC, SG_DEBUG, "got file " << fileName << " in " << directory->absolutePath());
 526             } else {
 527                 directory->didFailToUpdateFile(fileName);
 528             }
 529
 530             directory->repository()->finishedRequest(this);
 531         }
 532     private:
 533         static std::string makeUrl(HTTPDirectory* d, const std::string& file)
 534         {
 535             return d->url() + "/" + file;
 536         }
 537
 538         SGPath pathInRepo() const
 539         {
 540             SGPath p(directory->absolutePath());
 541             p.append(fileName);
 542             return p;
 543         }
 544
 545         HTTPDirectory* directory;
 546         std::string fileName; // if empty, we're getting the directory itself
 547         simgear::sha1nfo hashContext;
 548         int fd;
 549     };
 550
 551     class DirGetRequest : public HTTP::Request
 552     {
 553     public:
 554         DirGetRequest(HTTPDirectory* d) :
 555             HTTP::Request(makeUrl(d)),
 556             directory(d),
 557             _isRootDir(false)
 558         {
 559             sha1_init(&hashContext);
 560             SG_LOG(SG_TERRASYNC, SG_INFO, "will GET dir " << url());
 561
 562         }
 563
 564         void setIsRootDir()
 565         {
 566             _isRootDir = true;
 567         }
 568
 569         bool isRootDir() const
 570         {
 571             return _isRootDir;
 572         }
 573
 574     protected:
 575         virtual void gotBodyData(const char* s, int n)
 576         {
 577             body += std::string(s, n);
 578             sha1_write(&hashContext, s, n);
 579         }
 580
 581         virtual void onDone()
 582         {
 583             if (responseCode() == 200) {
 584                 std::string hash = strutils::encodeHex((char*) sha1_result(&hashContext));
 585                 std::string curHash = directory->repository()->hashForPath(path());
 586                 if (hash != curHash) {
 587
 588                     simgear::Dir d(directory->absolutePath());
 589                     if (!d.exists()) {
 590                         if (!d.create(0700)) {
 591                             throw sg_io_exception("Unable to create directory", d.path());
 592                         }
 593                     }
 594
 595                     // dir index data has changed, so write to disk and update
 596                     // the hash accordingly
 597                     std::ofstream of(pathInRepo().str().c_str(), std::ios::trunc | std::ios::out);
 598                     assert(of.is_open());
 599                     of.write(body.data(), body.size());
 600                     of.close();
 601                     directory->dirIndexUpdated(hash);
 602
 603                     SG_LOG(SG_TERRASYNC, SG_DEBUG, "updated dir index " << directory->absolutePath());
 604
 605                 }
 606
 607                 // either way we've confirmed the index is valid so update
 608                 // children now
 609                 directory->updateChildrenBasedOnHash();
 610             } else {
 611                 directory->failedToUpdate();
 612             }
 613
 614             directory->repository()->finishedRequest(this);
 615         }
 616     private:
 617         static std::string makeUrl(HTTPDirectory* d)
 618         {
 619             return d->url() + "/.dirindex";
 620         }
 621
 622         SGPath pathInRepo() const
 623         {
 624             SGPath p(directory->absolutePath());
 625             p.append(".dirindex");
 626             return p;
 627         }
 628
 629         HTTPDirectory* directory;
 630         simgear::sha1nfo hashContext;
 631         std::string body;
 632         bool _isRootDir; ///< is this the repository root?
 633     };
 634
 635
 636     HTTP::Request_ptr HTTPRepoPrivate::updateFile(HTTPDirectory* dir, const std::string& name)
 637     {
 638         HTTP::Request_ptr r(new FileGetRequest(dir, name));
 639         http->makeRequest(r);
 640         requests.push_back(r);
 641         return r;
 642     }
 643
 644     HTTP::Request_ptr HTTPRepoPrivate::updateDir(HTTPDirectory* dir)
 645     {
 646         HTTP::Request_ptr r(new DirGetRequest(dir));
 647         http->makeRequest(r);
 648         requests.push_back(r);
 649         return r;
 650     }
 651
 652
 653     class HashEntryWithPath
 654     {
 655     public:
 656         HashEntryWithPath(const std::string& p) : path(p) {}
 657         bool operator()(const HTTPRepoPrivate::HashCacheEntry& entry) const
 658         { return entry.filePath == path; }
 659     private:
 660         std::string path;
 661     };
 662
 663     std::string HTTPRepoPrivate::hashForPath(const SGPath& p)
 664     {
 665         HashCache::iterator it = std::find_if(hashes.begin(), hashes.end(), HashEntryWithPath(p.str()));
 666         if (it != hashes.end()) {
 667             // ensure data on disk hasn't changed.
 668             // we could also use the file type here if we were paranoid
 669             if ((p.sizeInBytes() == it->lengthBytes) && (p.modTime() == it->modTime)) {
 670                 return it->hashHex;
 671             }
 672
 673             // entry in the cache, but it's stale so remove and fall through
 674             hashes.erase(it);
 675         }
 676
 677         std::string hash = computeHashForPath(p);
 678         updatedFileContents(p, hash);
 679         return hash;
 680     }
 681
 682     std::string HTTPRepoPrivate::computeHashForPath(const SGPath& p)
 683     {
 684         if (!p.exists())
 685             return std::string();
 686         sha1nfo info;
 687         sha1_init(&info);
 688         char* buf = static_cast<char*>(malloc(1024 * 1024));
 689         size_t readLen;
 690         int fd = ::open(p.c_str(), O_RDONLY);
 691         if (fd < 0) {
 692             throw sg_io_exception("Couldn't open file for compute hash", p);
 693         }
 694         while ((readLen = ::read(fd, buf, 1024 * 1024)) > 0) {
 695             sha1_write(&info, buf, readLen);
 696         }
 697
 698         ::close(fd);
 699         free(buf);
 700         std::string hashBytes((char*) sha1_result(&info), HASH_LENGTH);
 701         return strutils::encodeHex(hashBytes);
 702     }
 703
 704     void HTTPRepoPrivate::updatedFileContents(const SGPath& p, const std::string& newHash)
 705     {
 706         // remove the existing entry
 707         HashCache::iterator it = std::find_if(hashes.begin(), hashes.end(), HashEntryWithPath(p.str()));
 708         if (it != hashes.end()) {
 709             hashes.erase(it);
 710         }
 711
 712         if (newHash.empty()) {
 713             return; // we're done
 714         }
 715
 716         // use a cloned SGPath and reset its caching to force one stat() call
 717         SGPath p2(p);
 718         p2.set_cached(false);
 719         p2.set_cached(true);
 720
 721         HashCacheEntry entry;
 722         entry.filePath = p.str();
 723         entry.hashHex = newHash;
 724         entry.modTime = p2.modTime();
 725         entry.lengthBytes = p2.sizeInBytes();
 726         hashes.push_back(entry);
 727
 728         writeHashCache();
 729     }
 730
 731     void HTTPRepoPrivate::writeHashCache()
 732     {
 733         SGPath cachePath = basePath;
 734         cachePath.append(".hashes");
 735
 736         std::ofstream stream(cachePath.str().c_str(),std::ios::out | std::ios::trunc);
 737         HashCache::const_iterator it;
 738         for (it = hashes.begin(); it != hashes.end(); ++it) {
 739             stream << it->filePath << ":" << it->modTime << ":"
 740             << it->lengthBytes << ":" << it->hashHex << "\n";
 741         }
 742         stream.close();
 743     }
 744
 745     void HTTPRepoPrivate::parseHashCache()
 746     {
 747         hashes.clear();
 748         SGPath cachePath = basePath;
 749         cachePath.append(".hashes");
 750         if (!cachePath.exists()) {
 751             return;
 752         }
 753
 754         std::ifstream stream(cachePath.str().c_str(), std::ios::in);
 755         char buf[2048];
 756         char* lastToken;
 757
 758         while (!stream.eof()) {
 759             stream.getline(buf, 2048);
 760             lastToken = 0;
 761             char* nameData = ::strtok_r(buf, ":", &lastToken);
 762             char* timeData = ::strtok_r(NULL, ":", &lastToken);
 763             char* sizeData = ::strtok_r(NULL, ":", &lastToken);
 764             char* hashData = ::strtok_r(NULL, ":", &lastToken);
 765             if (!nameData || !timeData || !sizeData || !hashData) {
 766                 continue;
 767             }
 768
 769             HashCacheEntry entry;
 770             entry.filePath = nameData;
 771             entry.hashHex = hashData;
 772             entry.modTime = strtol(timeData, NULL, 10);
 773             entry.lengthBytes = strtol(sizeData, NULL, 10);
 774             hashes.push_back(entry);
 775         }
 776     }
 777
 778     class DirectoryWithPath
 779     {
 780     public:
 781         DirectoryWithPath(const std::string& p) : path(p) {}
 782         bool operator()(const HTTPDirectory* entry) const
 783         { return entry->relativePath().str() == path; }
 784     private:
 785         std::string path;
 786     };
 787
 788     HTTPDirectory* HTTPRepoPrivate::getOrCreateDirectory(const std::string& path)
 789     {
 790         DirectoryWithPath p(path);
 791         DirectoryVector::iterator it = std::find_if(directories.begin(), directories.end(), p);
 792         if (it != directories.end()) {
 793             return *it;
 794         }
 795
 796         HTTPDirectory* d = new HTTPDirectory(this, path);
 797         directories.push_back(d);
 798         return d;
 799     }
 800
 801     bool HTTPRepoPrivate::deleteDirectory(const std::string& path)
 802     {
 803         DirectoryWithPath p(path);
 804         DirectoryVector::iterator it = std::find_if(directories.begin(), directories.end(), p);
 805         if (it != directories.end()) {
 806             HTTPDirectory* d = *it;
 807             directories.erase(it);
 808             Dir dir(d->absolutePath());
 809             bool result = dir.remove(true);
 810             delete d;
 811
 812             // update the hash cache too
 813             updatedFileContents(path, std::string());
 814
 815             return result;
 816         }
 817
 818         return false;
 819     }
 820
 821     void HTTPRepoPrivate::finishedRequest(const HTTP::Request_ptr& req)
 822     {
 823         RequestVector::iterator it = std::find(requests.begin(), requests.end(), req);
 824         if (it == requests.end()) {
 825             throw sg_exception("lost request somehow");
 826         }
 827         requests.erase(it);
 828         if (requests.empty()) {
 829             isUpdating = false;
 830         }
 831     }
 832
 833     void HTTPRepoPrivate::failedToGetRootIndex()
 834     {
 835         SG_LOG(SG_TERRASYNC, SG_WARN, "Failed to get root of repo:" << baseUrl);
 836         status = AbstractRepository::REPO_ERROR_NOT_FOUND;
 837     }
 838
 839
 840 } // of namespace simgear