simgear/io/HTTPRepository.cxx

   1 // HTTPRepository.cxx -- plain HTTP TerraSync remote client
   2 //
   3 // Copyright (C) 20126  James Turner <zakalawe@mac.com>
   4 //
   5 // This program is free software; you can redistribute it and/or
   6 // modify it under the terms of the GNU General Public License as
   7 // published by the Free Software Foundation; either version 2 of the
   8 // License, or (at your option) any later version.
   9 //
  10 // This program is distributed in the hope that it will be useful, but
  11 // WITHOUT ANY WARRANTY; without even the implied warranty of
  12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 // General Public License for more details.
  14 //
  15 // You should have received a copy of the GNU General Public License
  16 // along with this program; if not, write to the Free Software
  17 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  18
  19 #include "HTTPRepository.hxx"
  20
  21 #include <simgear_config.h>
  22
  23 #include <iostream>
  24 #include <cstring>
  25 #include <cassert>
  26 #include <algorithm>
  27 #include <sstream>
  28 #include <map>
  29 #include <set>
  30 #include <fstream>
  31 #include <limits>
  32 #include <cstdlib>
  33
  34 #include <fcntl.h>
  35
  36 #include "simgear/debug/logstream.hxx"
  37 #include "simgear/misc/strutils.hxx"
  38 #include <simgear/misc/sg_dir.hxx>
  39 #include <simgear/io/HTTPClient.hxx>
  40 #include <simgear/io/sg_file.hxx>
  41 #include <simgear/misc/sgstream.hxx>
  42 #include <simgear/structure/exception.hxx>
  43 #include <simgear/timing/timestamp.hxx>
  44
  45 #include <simgear/misc/sg_hash.hxx>
  46
  47 #if defined(SG_WINDOWS)
  48
  49 /*
  50  * public domain strtok_r() by Charlie Gordon
  51  *
  52  *   from comp.lang.c  9/14/2007
  53  *
  54  *      http://groups.google.com/group/comp.lang.c/msg/2ab1ecbb86646684
  55  *
  56  *     (Declaration that it's public domain):
  57  *      http://groups.google.com/group/comp.lang.c/msg/7c7b39328fefab9c
  58  */
  59
  60 char* strtok_r(
  61     char *str,
  62     const char *delim,
  63     char **nextp)
  64 {
  65     char *ret;
  66
  67     if (str == NULL)
  68     {
  69         str = *nextp;
  70     }
  71
  72     str += strspn(str, delim);
  73
  74     if (*str == '\0')
  75     {
  76         return NULL;
  77     }
  78
  79     ret = str;
  80
  81     str += strcspn(str, delim);
  82
  83     if (*str)
  84     {
  85         *str++ = '\0';
  86     }
  87
  88     *nextp = str;
  89
  90     return ret;
  91 }
  92 #endif
  93
  94 namespace simgear
  95 {
  96
  97     class HTTPDirectory;
  98
  99     class HTTPRepoGetRequest : public HTTP::Request
 100     {
 101     public:
 102         HTTPRepoGetRequest(HTTPDirectory* d, const std::string& u) :
 103             HTTP::Request(u),
 104             _directory(d)
 105         {
 106         }
 107
 108         virtual void cancel();
 109
 110         size_t contentSize() const
 111         {
 112             return _contentSize;
 113         }
 114
 115         void setContentSize(size_t sz)
 116         {
 117             _contentSize = sz;
 118         }
 119     protected:
 120         HTTPDirectory* _directory;
 121         size_t _contentSize;
 122     };
 123
 124     typedef SGSharedPtr<HTTPRepoGetRequest> RepoRequestPtr;
 125
 126 class HTTPRepoPrivate
 127 {
 128 public:
 129     struct HashCacheEntry
 130     {
 131         std::string filePath;
 132         time_t modTime;
 133         size_t lengthBytes;
 134         std::string hashHex;
 135
 136     };
 137
 138     typedef std::vector<HashCacheEntry> HashCache;
 139     HashCache hashes;
 140     bool hashCacheDirty;
 141
 142     struct Failure
 143     {
 144         SGPath path;
 145         AbstractRepository::ResultCode error;
 146     };
 147
 148     typedef std::vector<Failure> FailureList;
 149     FailureList failures;
 150
 151     HTTPRepoPrivate(HTTPRepository* parent) :
 152         hashCacheDirty(false),
 153         p(parent),
 154         isUpdating(false),
 155         status(AbstractRepository::REPO_NO_ERROR),
 156         totalDownloaded(0)
 157     { ; }
 158
 159     ~HTTPRepoPrivate();
 160
 161     HTTPRepository* p; // link back to outer
 162     HTTP::Client* http;
 163     std::string baseUrl;
 164     SGPath basePath;
 165     bool isUpdating;
 166     AbstractRepository::ResultCode status;
 167     HTTPDirectory* rootDir;
 168     size_t totalDownloaded;
 169
 170     HTTP::Request_ptr updateFile(HTTPDirectory* dir, const std::string& name,
 171                                  size_t sz);
 172     HTTP::Request_ptr updateDir(HTTPDirectory* dir, const std::string& hash,
 173                                 size_t sz);
 174
 175     std::string hashForPath(const SGPath& p);
 176     void updatedFileContents(const SGPath& p, const std::string& newHash);
 177     void parseHashCache();
 178     std::string computeHashForPath(const SGPath& p);
 179     void writeHashCache();
 180
 181     void failedToGetRootIndex(AbstractRepository::ResultCode st);
 182     void failedToUpdateChild(const SGPath& relativePath,
 183                              AbstractRepository::ResultCode fileStatus);
 184
 185     typedef std::vector<RepoRequestPtr> RequestVector;
 186     RequestVector queuedRequests,
 187         activeRequests;
 188
 189     void makeRequest(RepoRequestPtr req);
 190     void finishedRequest(const RepoRequestPtr& req);
 191
 192     HTTPDirectory* getOrCreateDirectory(const std::string& path);
 193     bool deleteDirectory(const std::string& path);
 194
 195     typedef std::vector<HTTPDirectory*> DirectoryVector;
 196     DirectoryVector directories;
 197
 198 };
 199
 200 class HTTPDirectory
 201 {
 202     struct ChildInfo
 203     {
 204         enum Type
 205         {
 206             FileType,
 207             DirectoryType
 208         };
 209
 210         ChildInfo(Type ty, const char* nameData, const char* hashData) :
 211             type(ty),
 212             name(nameData),
 213             hash(hashData ? hashData : ""),
 214             sizeInBytes(0)
 215         {
 216         }
 217
 218         ChildInfo(const ChildInfo& other) :
 219             type(other.type),
 220             name(other.name),
 221             hash(other.hash),
 222             sizeInBytes(other.sizeInBytes)
 223         { }
 224
 225         void setSize(const char* sizeData)
 226         {
 227             sizeInBytes = ::strtol(sizeData, NULL, 10);
 228         }
 229
 230         bool operator<(const ChildInfo& other) const
 231         {
 232             return name < other.name;
 233         }
 234
 235         Type type;
 236         std::string name, hash;
 237         size_t sizeInBytes;
 238     };
 239
 240     typedef std::vector<ChildInfo> ChildInfoList;
 241     ChildInfoList children;
 242
 243 public:
 244     HTTPDirectory(HTTPRepoPrivate* repo, const std::string& path) :
 245         _repository(repo),
 246         _relativePath(path)
 247   {
 248       assert(repo);
 249
 250       SGPath p(absolutePath());
 251       if (p.exists()) {
 252           try {
 253               // already exists on disk
 254               parseDirIndex(children);
 255               std::sort(children.begin(), children.end());
 256           } catch (sg_exception& ) {
 257               // parsing cache failed
 258               children.clear();
 259           }
 260       }
 261   }
 262
 263     HTTPRepoPrivate* repository() const
 264     {
 265         return _repository;
 266     }
 267
 268     std::string url() const
 269     {
 270         if (_relativePath.str().empty()) {
 271             return _repository->baseUrl;
 272         }
 273
 274         return _repository->baseUrl + "/" + _relativePath.str();
 275     }
 276
 277     void dirIndexUpdated(const std::string& hash)
 278     {
 279         SGPath fpath(_relativePath);
 280         fpath.append(".dirindex");
 281         _repository->updatedFileContents(fpath, hash);
 282
 283         children.clear();
 284         parseDirIndex(children);
 285         std::sort(children.begin(), children.end());
 286     }
 287
 288     void failedToUpdate(AbstractRepository::ResultCode status)
 289     {
 290         if (_relativePath.isNull()) {
 291             // root dir failed
 292             _repository->failedToGetRootIndex(status);
 293         } else {
 294             _repository->failedToUpdateChild(_relativePath, status);
 295         }
 296     }
 297
 298     void updateChildrenBasedOnHash()
 299     {
 300         //SG_LOG(SG_TERRASYNC, SG_DEBUG, "updated children for:" << relativePath());
 301
 302         string_list indexNames = indexChildren(),
 303             toBeUpdated, orphans;
 304         simgear::Dir d(absolutePath());
 305         PathList fsChildren = d.children(0);
 306         PathList::const_iterator it = fsChildren.begin();
 307
 308         for (; it != fsChildren.end(); ++it) {
 309             SG_LOG(SG_TERRASYNC, SG_DEBUG, "processing child: '" << it->str() << "', isDir=" << it->isDir() );
 310             ChildInfo info(it->isDir() ? ChildInfo::DirectoryType : ChildInfo::FileType,
 311                            it->file().c_str(), NULL);
 312             std::string hash = hashForChild(info);
 313             SG_LOG(SG_TERRASYNC, SG_DEBUG, "hash is: '" << hash << "'" );
 314
 315             ChildInfoList::iterator c = findIndexChild(it->file());
 316             if (c == children.end()) {
 317                 SG_LOG(SG_TERRASYNC, SG_DEBUG, "is orphan '" << it->file() << "'" );
 318                 orphans.push_back(it->file());
 319             } else if (c->hash != hash) {
 320                 SG_LOG(SG_TERRASYNC, SG_DEBUG, "hash mismatch'" << it->file() << "', c->name=" << c->name );
 321                 // file exists, but hash mismatch, schedule update
 322                 if (!hash.empty()) {
 323                     SG_LOG(SG_TERRASYNC, SG_DEBUG, "file exists but hash is wrong for:" << c->name);
 324                     SG_LOG(SG_TERRASYNC, SG_DEBUG, "on disk:" << hash << " vs in info:" << c->hash);
 325                 }
 326
 327                 toBeUpdated.push_back(c->name);
 328             } else {
 329                 // file exists and hash is valid. If it's a directory,
 330                 // perform a recursive check.
 331                 SG_LOG(SG_TERRASYNC, SG_DEBUG, "file exists hash is good:" << c->name);
 332                 if (c->type == ChildInfo::DirectoryType) {
 333                     SG_LOG(SG_TERRASYNC, SG_DEBUG, "going recursive for:" << c->name);
 334                     SGPath p(relativePath());
 335                     p.append(c->name);
 336                     HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str());
 337                     childDir->updateChildrenBasedOnHash();
 338                 }
 339             }
 340
 341             // remove existing file system children from the index list,
 342             // so we can detect new children
 343             SG_LOG(SG_TERRASYNC, SG_DEBUG, "looking for name in indexNames:" << c->name);
 344             string_list::iterator it = std::find(indexNames.begin(), indexNames.end(), c->name);
 345             if (it != indexNames.end()) {
 346                 SG_LOG(SG_TERRASYNC, SG_DEBUG, "found name in indexNames, erasing:" << c->name);
 347                 indexNames.erase(it);
 348             }
 349         } // of real children iteration
 350
 351         // all remaining names in indexChilden are new children
 352         toBeUpdated.insert(toBeUpdated.end(), indexNames.begin(), indexNames.end());
 353
 354         removeOrphans(orphans);
 355         scheduleUpdates(toBeUpdated);
 356     }
 357
 358     void removeOrphans(const string_list& orphans)
 359     {
 360         string_list::const_iterator it;
 361         for (it = orphans.begin(); it != orphans.end(); ++it) {
 362             removeChild(*it);
 363         }
 364     }
 365
 366     string_list indexChildren() const
 367     {
 368         string_list r;
 369         r.reserve(children.size());
 370         ChildInfoList::const_iterator it;
 371         for (it=children.begin(); it != children.end(); ++it) {
 372             r.push_back(it->name);
 373         }
 374         return r;
 375     }
 376
 377     void scheduleUpdates(const string_list& names)
 378     {
 379         string_list::const_iterator it;
 380         for (it = names.begin(); it != names.end(); ++it) {
 381             ChildInfoList::iterator cit = findIndexChild(*it);
 382             if (cit == children.end()) {
 383                 SG_LOG(SG_TERRASYNC, SG_WARN, "scheduleUpdate, unknown child:" << *it);
 384                 continue;
 385             }
 386
 387             if (cit->type == ChildInfo::FileType) {
 388                 _repository->updateFile(this, *it, cit->sizeInBytes);
 389             } else {
 390                 SGPath p(relativePath());
 391                 p.append(*it);
 392                 HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str());
 393                 _repository->updateDir(childDir, cit->hash, cit->sizeInBytes);
 394             }
 395         }
 396     }
 397
 398     SGPath absolutePath() const
 399     {
 400         SGPath r(_repository->basePath);
 401         r.append(_relativePath.str());
 402         return r;
 403     }
 404
 405     SGPath relativePath() const
 406     {
 407         return _relativePath;
 408     }
 409
 410     void didUpdateFile(const std::string& file, const std::string& hash, size_t sz)
 411     {
 412         // check hash matches what we expected
 413         ChildInfoList::iterator it = findIndexChild(file);
 414         if (it == children.end()) {
 415             SG_LOG(SG_TERRASYNC, SG_WARN, "updated file but not found in dir:" << _relativePath << " " << file);
 416         } else {
 417             SGPath fpath(_relativePath);
 418             fpath.append(file);
 419
 420             if (it->hash != hash) {
 421                 _repository->failedToUpdateChild(_relativePath, AbstractRepository::REPO_ERROR_CHECKSUM);
 422             } else {
 423                 _repository->updatedFileContents(fpath, hash);
 424                 _repository->totalDownloaded += sz;
 425                 //SG_LOG(SG_TERRASYNC, SG_INFO, "did update:" << fpath);
 426             } // of hash matches
 427         } // of found in child list
 428     }
 429
 430     void didFailToUpdateFile(const std::string& file,
 431                              AbstractRepository::ResultCode status)
 432     {
 433         SGPath fpath(_relativePath);
 434         fpath.append(file);
 435         _repository->failedToUpdateChild(fpath, status);
 436     }
 437 private:
 438
 439     struct ChildWithName
 440     {
 441         ChildWithName(const std::string& n) : name(n) {}
 442         std::string name;
 443
 444         bool operator()(const ChildInfo& info) const
 445         { return info.name == name; }
 446     };
 447
 448     ChildInfoList::iterator findIndexChild(const std::string& name)
 449     {
 450         return std::find_if(children.begin(), children.end(), ChildWithName(name));
 451     }
 452
 453     bool parseDirIndex(ChildInfoList& children)
 454     {
 455         SGPath p(absolutePath());
 456         p.append(".dirindex");
 457         if (!p.exists()) {
 458             return false;
 459         }
 460
 461         std::ifstream indexStream( p.c_str(), std::ios::in );
 462
 463         if ( !indexStream.is_open() ) {
 464             throw sg_io_exception("cannot open dirIndex file", p);
 465         }
 466
 467         char lineBuffer[512];
 468         char* lastToken;
 469
 470         while (!indexStream.eof() ) {
 471             indexStream.getline(lineBuffer, 512);
 472             lastToken = 0;
 473             char* typeData = ::strtok_r(lineBuffer, ":", &lastToken);
 474             if (!typeData) {
 475                 continue; // skip blank line
 476             }
 477
 478             if (!typeData) {
 479                 // malformed entry
 480                 throw sg_io_exception("Malformed dir index file", p);
 481             }
 482
 483             if (!strcmp(typeData, "version")) {
 484                 continue;
 485             } else if (!strcmp(typeData, "path")) {
 486                 continue;
 487             }
 488
 489             char* nameData = ::strtok_r(NULL, ":", &lastToken);
 490             char* hashData = ::strtok_r(NULL, ":", &lastToken);
 491             char* sizeData = ::strtok_r(NULL, ":", &lastToken);
 492
 493             if (typeData[0] == 'f') {
 494                 children.push_back(ChildInfo(ChildInfo::FileType, nameData, hashData));
 495             } else if (typeData[0] == 'd') {
 496                 children.push_back(ChildInfo(ChildInfo::DirectoryType, nameData, hashData));
 497             } else {
 498                 throw sg_io_exception("Malformed line code in dir index file", p);
 499             }
 500
 501             if (sizeData) {
 502                 children.back().setSize(sizeData);
 503             }
 504         }
 505
 506         return true;
 507     }
 508
 509     void removeChild(const std::string& name)
 510     {
 511         SGPath p(absolutePath());
 512         p.append(name);
 513         bool ok;
 514
 515         SGPath fpath(_relativePath);
 516         fpath.append(name);
 517
 518         if (p.isDir()) {
 519             ok = _repository->deleteDirectory(fpath.str());
 520         } else {
 521             // remove the hash cache entry
 522             _repository->updatedFileContents(fpath, std::string());
 523             ok = p.remove();
 524         }
 525
 526         if (!ok) {
 527             SG_LOG(SG_TERRASYNC, SG_WARN, "removal failed for:" << p);
 528             throw sg_io_exception("Failed to remove existing file/dir:", p);
 529         }
 530     }
 531
 532     std::string hashForChild(const ChildInfo& child) const
 533     {
 534         SGPath p(absolutePath());
 535         p.append(child.name);
 536         if (child.type == ChildInfo::DirectoryType) {
 537             p.append(".dirindex");
 538         }
 539         return _repository->hashForPath(p);
 540     }
 541
 542   HTTPRepoPrivate* _repository;
 543   SGPath _relativePath; // in URL and file-system space
 544
 545
 546 };
 547
 548 HTTPRepository::HTTPRepository(const SGPath& base, HTTP::Client *cl) :
 549     _d(new HTTPRepoPrivate(this))
 550 {
 551     _d->http = cl;
 552     _d->basePath = base;
 553     _d->rootDir = new HTTPDirectory(_d.get(), "");
 554     _d->parseHashCache();
 555 }
 556
 557 HTTPRepository::~HTTPRepository()
 558 {
 559 }
 560
 561 void HTTPRepository::setBaseUrl(const std::string &url)
 562 {
 563   _d->baseUrl = url;
 564 }
 565
 566 std::string HTTPRepository::baseUrl() const
 567 {
 568   return _d->baseUrl;
 569 }
 570
 571 HTTP::Client* HTTPRepository::http() const
 572 {
 573   return _d->http;
 574 }
 575
 576 SGPath HTTPRepository::fsBase() const
 577 {
 578   return SGPath();
 579 }
 580
 581 void HTTPRepository::update()
 582 {
 583     if (_d->isUpdating) {
 584         return;
 585     }
 586
 587     _d->status = REPO_NO_ERROR;
 588     _d->isUpdating = true;
 589     _d->failures.clear();
 590     _d->updateDir(_d->rootDir, std::string(), 0);
 591 }
 592
 593 bool HTTPRepository::isDoingSync() const
 594 {
 595     if (_d->status != REPO_NO_ERROR) {
 596         return false;
 597     }
 598
 599     return _d->isUpdating;
 600 }
 601
 602 size_t HTTPRepository::bytesToDownload() const
 603 {
 604     size_t result = 0;
 605
 606     HTTPRepoPrivate::RequestVector::const_iterator r;
 607     for (r = _d->queuedRequests.begin(); r != _d->queuedRequests.end(); ++r) {
 608         result += (*r)->contentSize();
 609     }
 610
 611     for (r = _d->activeRequests.begin(); r != _d->activeRequests.end(); ++r) {
 612         result += (*r)->contentSize() - (*r)->responseBytesReceived();
 613     }
 614
 615     return result;
 616 }
 617
 618 size_t HTTPRepository::bytesDownloaded() const
 619 {
 620     size_t result = _d->totalDownloaded;
 621
 622     HTTPRepoPrivate::RequestVector::const_iterator r;
 623     for (r = _d->activeRequests.begin(); r != _d->activeRequests.end(); ++r) {
 624         result += (*r)->responseBytesReceived();
 625     }
 626
 627     return result;
 628 }
 629
 630 AbstractRepository::ResultCode
 631 HTTPRepository::failure() const
 632 {
 633     if ((_d->status == REPO_NO_ERROR) && !_d->failures.empty()) {
 634         return REPO_PARTIAL_UPDATE;
 635     }
 636
 637     return _d->status;
 638 }
 639
 640     void HTTPRepoGetRequest::cancel()
 641     {
 642         _directory->repository()->http->cancelRequest(this, "Reposiotry cancelled");
 643         _directory = 0;
 644     }
 645
 646     class FileGetRequest : public HTTPRepoGetRequest
 647     {
 648     public:
 649         FileGetRequest(HTTPDirectory* d, const std::string& file) :
 650             HTTPRepoGetRequest(d, makeUrl(d, file)),
 651             fileName(file)
 652         {
 653             pathInRepo = _directory->absolutePath();
 654             pathInRepo.append(fileName);
 655             //SG_LOG(SG_TERRASYNC, SG_INFO, "will GET file " << url());
 656         }
 657
 658     protected:
 659         virtual void gotBodyData(const char* s, int n)
 660         {
 661             if (!file.get()) {
 662                 file.reset(new SGFile(pathInRepo.str()));
 663                 if (!file->open(SG_IO_OUT)) {
 664                   SG_LOG(SG_TERRASYNC, SG_WARN, "unable to create file " << pathInRepo);
 665                   _directory->repository()->http->cancelRequest(this, "Unable to create output file");
 666                 }
 667
 668                 sha1_init(&hashContext);
 669             }
 670
 671             sha1_write(&hashContext, s, n);
 672             file->write(s, n);
 673         }
 674
 675         virtual void onDone()
 676         {
 677             file->close();
 678             if (responseCode() == 200) {
 679                 std::string hash = strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
 680                 _directory->didUpdateFile(fileName, hash, contentSize());
 681                 //SG_LOG(SG_TERRASYNC, SG_INFO, "got file " << fileName << " in " << _directory->absolutePath());
 682             } else if (responseCode() == 404) {
 683                 _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_FILE_NOT_FOUND);
 684             } else {
 685                 _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_HTTP);
 686             }
 687
 688             _directory->repository()->finishedRequest(this);
 689         }
 690
 691         virtual void onFail()
 692         {
 693             file.reset();
 694             if (pathInRepo.exists()) {
 695                 pathInRepo.remove();
 696             }
 697
 698             if (_directory) {
 699                 _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_SOCKET);
 700                 _directory->repository()->finishedRequest(this);
 701             }
 702         }
 703     private:
 704         static std::string makeUrl(HTTPDirectory* d, const std::string& file)
 705         {
 706             return d->url() + "/" + file;
 707         }
 708
 709         std::string fileName; // if empty, we're getting the directory itself
 710         SGPath pathInRepo;
 711         simgear::sha1nfo hashContext;
 712         std::auto_ptr<SGFile> file;
 713     };
 714
 715     class DirGetRequest : public HTTPRepoGetRequest
 716     {
 717     public:
 718         DirGetRequest(HTTPDirectory* d, const std::string& targetHash) :
 719             HTTPRepoGetRequest(d, makeUrl(d)),
 720             _isRootDir(false),
 721             _targetHash(targetHash)
 722         {
 723             sha1_init(&hashContext);
 724            //SG_LOG(SG_TERRASYNC, SG_INFO, "will GET dir " << url());
 725         }
 726
 727         void setIsRootDir()
 728         {
 729             _isRootDir = true;
 730         }
 731
 732         bool isRootDir() const
 733         {
 734             return _isRootDir;
 735         }
 736
 737     protected:
 738         virtual void gotBodyData(const char* s, int n)
 739         {
 740             body += std::string(s, n);
 741             sha1_write(&hashContext, s, n);
 742         }
 743
 744         virtual void onDone()
 745         {
 746             if (responseCode() == 200) {
 747                 std::string hash = strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
 748                 if (!_targetHash.empty() && (hash != _targetHash)) {
 749                     _directory->failedToUpdate(AbstractRepository::REPO_ERROR_CHECKSUM);
 750                     _directory->repository()->finishedRequest(this);
 751                     return;
 752                 }
 753
 754                 std::string curHash = _directory->repository()->hashForPath(path());
 755                 if (hash != curHash) {
 756                     simgear::Dir d(_directory->absolutePath());
 757                     if (!d.exists()) {
 758                         if (!d.create(0700)) {
 759                             throw sg_io_exception("Unable to create directory", d.path());
 760                         }
 761                     }
 762
 763                     // dir index data has changed, so write to disk and update
 764                     // the hash accordingly
 765                     std::ofstream of(pathInRepo().c_str(), std::ios::trunc | std::ios::out);
 766                     if (!of.is_open()) {
 767                         throw sg_io_exception("Failed to open directory index file for writing", pathInRepo().c_str());
 768                     }
 769
 770                     of.write(body.data(), body.size());
 771                     of.close();
 772                     _directory->dirIndexUpdated(hash);
 773
 774                     //SG_LOG(SG_TERRASYNC, SG_INFO, "updated dir index " << _directory->absolutePath());
 775                 }
 776
 777                 _directory->repository()->totalDownloaded += contentSize();
 778
 779                 try {
 780                     // either way we've confirmed the index is valid so update
 781                     // children now
 782                     SGTimeStamp st;
 783                     st.stamp();
 784                     _directory->updateChildrenBasedOnHash();
 785                     SG_LOG(SG_TERRASYNC, SG_INFO, "after update of:" << _directory->absolutePath() << " child update took:" << st.elapsedMSec());
 786                 } catch (sg_exception& ) {
 787                     _directory->failedToUpdate(AbstractRepository::REPO_ERROR_IO);
 788                 }
 789             } else if (responseCode() == 404) {
 790                 _directory->failedToUpdate(AbstractRepository::REPO_ERROR_FILE_NOT_FOUND);
 791             } else {
 792                 _directory->failedToUpdate(AbstractRepository::REPO_ERROR_HTTP);
 793             }
 794
 795             _directory->repository()->finishedRequest(this);
 796         }
 797
 798         virtual void onFail()
 799         {
 800             if (_directory) {
 801                 _directory->failedToUpdate(AbstractRepository::REPO_ERROR_SOCKET);
 802                 _directory->repository()->finishedRequest(this);
 803             }
 804         }
 805     private:
 806         static std::string makeUrl(HTTPDirectory* d)
 807         {
 808             return d->url() + "/.dirindex";
 809         }
 810
 811         SGPath pathInRepo() const
 812         {
 813             SGPath p(_directory->absolutePath());
 814             p.append(".dirindex");
 815             return p;
 816         }
 817
 818         simgear::sha1nfo hashContext;
 819         std::string body;
 820         bool _isRootDir; ///< is this the repository root?
 821         std::string _targetHash;
 822     };
 823
 824     HTTPRepoPrivate::~HTTPRepoPrivate()
 825     {
 826         DirectoryVector::iterator it;
 827         for (it=directories.begin(); it != directories.end(); ++it) {
 828             delete *it;
 829         }
 830
 831         RequestVector::iterator r;
 832         for (r=activeRequests.begin(); r != activeRequests.end(); ++r) {
 833             (*r)->cancel();
 834         }
 835     }
 836
 837     HTTP::Request_ptr HTTPRepoPrivate::updateFile(HTTPDirectory* dir, const std::string& name, size_t sz)
 838     {
 839         RepoRequestPtr r(new FileGetRequest(dir, name));
 840         r->setContentSize(sz);
 841         makeRequest(r);
 842         return r;
 843     }
 844
 845     HTTP::Request_ptr HTTPRepoPrivate::updateDir(HTTPDirectory* dir, const std::string& hash, size_t sz)
 846     {
 847         RepoRequestPtr r(new DirGetRequest(dir, hash));
 848         r->setContentSize(sz);
 849         makeRequest(r);
 850         return r;
 851     }
 852
 853
 854     class HashEntryWithPath
 855     {
 856     public:
 857         HashEntryWithPath(const std::string& p) : path(p) {}
 858         bool operator()(const HTTPRepoPrivate::HashCacheEntry& entry) const
 859         { return entry.filePath == path; }
 860     private:
 861         std::string path;
 862     };
 863
 864     std::string HTTPRepoPrivate::hashForPath(const SGPath& p)
 865     {
 866         HashCache::iterator it = std::find_if(hashes.begin(), hashes.end(), HashEntryWithPath(p.str()));
 867         if (it != hashes.end()) {
 868             // ensure data on disk hasn't changed.
 869             // we could also use the file type here if we were paranoid
 870             if ((p.sizeInBytes() == it->lengthBytes) && (p.modTime() == it->modTime)) {
 871                 return it->hashHex;
 872             }
 873
 874             // entry in the cache, but it's stale so remove and fall through
 875             hashes.erase(it);
 876         }
 877
 878         std::string hash = computeHashForPath(p);
 879         updatedFileContents(p, hash);
 880         return hash;
 881     }
 882
 883     std::string HTTPRepoPrivate::computeHashForPath(const SGPath& p)
 884     {
 885         if (!p.exists())
 886             return std::string();
 887         sha1nfo info;
 888         sha1_init(&info);
 889         char* buf = static_cast<char*>(malloc(1024 * 1024));
 890         size_t readLen;
 891         SGFile f(p.str());
 892         if (!f.open(SG_IO_IN)) {
 893             throw sg_io_exception("Couldn't open file for compute hash", p);
 894         }
 895         while ((readLen = f.read(buf, 1024 * 1024)) > 0) {
 896             sha1_write(&info, buf, readLen);
 897         }
 898
 899         f.close();
 900         free(buf);
 901         std::string hashBytes((char*) sha1_result(&info), HASH_LENGTH);
 902         return strutils::encodeHex(hashBytes);
 903     }
 904
 905     void HTTPRepoPrivate::updatedFileContents(const SGPath& p, const std::string& newHash)
 906     {
 907         // remove the existing entry
 908         HashCache::iterator it = std::find_if(hashes.begin(), hashes.end(), HashEntryWithPath(p.str()));
 909         if (it != hashes.end()) {
 910             hashes.erase(it);
 911             hashCacheDirty = true;
 912         }
 913
 914         if (newHash.empty()) {
 915             return; // we're done
 916         }
 917
 918         // use a cloned SGPath and reset its caching to force one stat() call
 919         SGPath p2(p);
 920         p2.set_cached(false);
 921         p2.set_cached(true);
 922
 923         HashCacheEntry entry;
 924         entry.filePath = p.str();
 925         entry.hashHex = newHash;
 926         entry.modTime = p2.modTime();
 927         entry.lengthBytes = p2.sizeInBytes();
 928         hashes.push_back(entry);
 929
 930         hashCacheDirty = true;
 931     }
 932
 933     void HTTPRepoPrivate::writeHashCache()
 934     {
 935         if (!hashCacheDirty) {
 936             return;
 937         }
 938
 939         SGPath cachePath = basePath;
 940         cachePath.append(".hashes");
 941
 942         std::ofstream stream(cachePath.c_str(),std::ios::out | std::ios::trunc);
 943         HashCache::const_iterator it;
 944         for (it = hashes.begin(); it != hashes.end(); ++it) {
 945             stream << it->filePath << ":" << it->modTime << ":"
 946             << it->lengthBytes << ":" << it->hashHex << "\n";
 947         }
 948         stream.close();
 949         hashCacheDirty = false;
 950     }
 951
 952     void HTTPRepoPrivate::parseHashCache()
 953     {
 954         hashes.clear();
 955         SGPath cachePath = basePath;
 956         cachePath.append(".hashes");
 957         if (!cachePath.exists()) {
 958             return;
 959         }
 960
 961         std::ifstream stream(cachePath.c_str(), std::ios::in);
 962         char buf[2048];
 963         char* lastToken;
 964
 965         while (!stream.eof()) {
 966             stream.getline(buf, 2048);
 967             lastToken = 0;
 968             char* nameData = ::strtok_r(buf, ":", &lastToken);
 969             char* timeData = ::strtok_r(NULL, ":", &lastToken);
 970             char* sizeData = ::strtok_r(NULL, ":", &lastToken);
 971             char* hashData = ::strtok_r(NULL, ":", &lastToken);
 972             if (!nameData || !timeData || !sizeData || !hashData) {
 973                 continue;
 974             }
 975
 976             HashCacheEntry entry;
 977             entry.filePath = nameData;
 978             entry.hashHex = hashData;
 979             entry.modTime = strtol(timeData, NULL, 10);
 980             entry.lengthBytes = strtol(sizeData, NULL, 10);
 981             hashes.push_back(entry);
 982         }
 983     }
 984
 985     class DirectoryWithPath
 986     {
 987     public:
 988         DirectoryWithPath(const std::string& p) : path(p) {}
 989         bool operator()(const HTTPDirectory* entry) const
 990         { return entry->relativePath().str() == path; }
 991     private:
 992         std::string path;
 993     };
 994
 995     HTTPDirectory* HTTPRepoPrivate::getOrCreateDirectory(const std::string& path)
 996     {
 997         DirectoryWithPath p(path);
 998         DirectoryVector::iterator it = std::find_if(directories.begin(), directories.end(), p);
 999         if (it != directories.end()) {
1000             return *it;
1001         }
1002
1003         HTTPDirectory* d = new HTTPDirectory(this, path);
1004         directories.push_back(d);
1005         return d;
1006     }
1007
1008     bool HTTPRepoPrivate::deleteDirectory(const std::string& path)
1009     {
1010         DirectoryWithPath p(path);
1011         DirectoryVector::iterator it = std::find_if(directories.begin(), directories.end(), p);
1012         if (it != directories.end()) {
1013             HTTPDirectory* d = *it;
1014             directories.erase(it);
1015             Dir dir(d->absolutePath());
1016             bool result = dir.remove(true);
1017             delete d;
1018
1019             // update the hash cache too
1020             updatedFileContents(path, std::string());
1021
1022             return result;
1023         }
1024
1025         return false;
1026     }
1027
1028     void HTTPRepoPrivate::makeRequest(RepoRequestPtr req)
1029     {
1030         if (activeRequests.size() > 4) {
1031             queuedRequests.push_back(req);
1032         } else {
1033             activeRequests.push_back(req);
1034             http->makeRequest(req);
1035         }
1036     }
1037
1038     void HTTPRepoPrivate::finishedRequest(const RepoRequestPtr& req)
1039     {
1040         RequestVector::iterator it = std::find(activeRequests.begin(), activeRequests.end(), req);
1041         if (it == activeRequests.end()) {
1042             throw sg_exception("lost request somehow", req->url());
1043         }
1044         activeRequests.erase(it);
1045
1046         if (!queuedRequests.empty()) {
1047             RepoRequestPtr rr = queuedRequests.front();
1048             queuedRequests.erase(queuedRequests.begin());
1049             activeRequests.push_back(rr);
1050             http->makeRequest(rr);
1051         }
1052
1053         writeHashCache();
1054
1055         if (activeRequests.empty() && queuedRequests.empty()) {
1056             isUpdating = false;
1057         }
1058     }
1059
1060     void HTTPRepoPrivate::failedToGetRootIndex(AbstractRepository::ResultCode st)
1061     {
1062         SG_LOG(SG_TERRASYNC, SG_WARN, "Failed to get root of repo:" << baseUrl);
1063         status = st;
1064     }
1065
1066     void HTTPRepoPrivate::failedToUpdateChild(const SGPath& relativePath,
1067                                               AbstractRepository::ResultCode fileStatus)
1068     {
1069         Failure f;
1070         f.path = relativePath;
1071         f.error = fileStatus;
1072         failures.push_back(f);
1073
1074         SG_LOG(SG_TERRASYNC, SG_WARN, "failed to update entry:" << relativePath << " code:" << fileStatus);
1075     }
1076
1077
1078
1079 } // of namespace simgear