simgear/io/HTTPRepository.cxx

   1 // HTTPRepository.cxx -- plain HTTP TerraSync remote client
   2 //
   3 // Copyright (C) 20126  James Turner <zakalawe@mac.com>
   4 //
   5 // This program is free software; you can redistribute it and/or
   6 // modify it under the terms of the GNU General Public License as
   7 // published by the Free Software Foundation; either version 2 of the
   8 // License, or (at your option) any later version.
   9 //
  10 // This program is distributed in the hope that it will be useful, but
  11 // WITHOUT ANY WARRANTY; without even the implied warranty of
  12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 // General Public License for more details.
  14 //
  15 // You should have received a copy of the GNU General Public License
  16 // along with this program; if not, write to the Free Software
  17 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  18
  19 #include "HTTPRepository.hxx"
  20
  21 #include <simgear_config.h>
  22
  23 #include <iostream>
  24 #include <cstring>
  25 #include <cassert>
  26 #include <algorithm>
  27 #include <sstream>
  28 #include <map>
  29 #include <set>
  30 #include <fstream>
  31 #include <limits>
  32 #include <cstdlib>
  33
  34 #include <fcntl.h>
  35
  36 #include "simgear/debug/logstream.hxx"
  37 #include "simgear/misc/strutils.hxx"
  38 #include <simgear/misc/sg_dir.hxx>
  39 #include <simgear/io/HTTPClient.hxx>
  40 #include <simgear/io/sg_file.hxx>
  41 #include <simgear/misc/sgstream.hxx>
  42 #include <simgear/structure/exception.hxx>
  43
  44 #include <simgear/misc/sg_hash.hxx>
  45
  46 #if defined(SG_WINDOWS)
  47
  48 /*
  49  * public domain strtok_r() by Charlie Gordon
  50  *
  51  *   from comp.lang.c  9/14/2007
  52  *
  53  *      http://groups.google.com/group/comp.lang.c/msg/2ab1ecbb86646684
  54  *
  55  *     (Declaration that it's public domain):
  56  *      http://groups.google.com/group/comp.lang.c/msg/7c7b39328fefab9c
  57  */
  58
  59 char* strtok_r(
  60     char *str,
  61     const char *delim,
  62     char **nextp)
  63 {
  64     char *ret;
  65
  66     if (str == NULL)
  67     {
  68         str = *nextp;
  69     }
  70
  71     str += strspn(str, delim);
  72
  73     if (*str == '\0')
  74     {
  75         return NULL;
  76     }
  77
  78     ret = str;
  79
  80     str += strcspn(str, delim);
  81
  82     if (*str)
  83     {
  84         *str++ = '\0';
  85     }
  86
  87     *nextp = str;
  88
  89     return ret;
  90 }
  91 #endif
  92
  93 namespace simgear
  94 {
  95
  96     class HTTPDirectory;
  97
  98     class HTTPRepoGetRequest : public HTTP::Request
  99     {
 100     public:
 101         HTTPRepoGetRequest(HTTPDirectory* d, const std::string& u) :
 102             HTTP::Request(u),
 103             _directory(d)
 104         {
 105         }
 106
 107         virtual void cancel()
 108         {
 109             _directory = 0;
 110             abort("Repository cancelled request");
 111         }
 112
 113         size_t contentSize() const
 114         {
 115             return _contentSize;
 116         }
 117
 118         void setContentSize(size_t sz)
 119         {
 120             _contentSize = sz;
 121         }
 122     protected:
 123         HTTPDirectory* _directory;
 124         size_t _contentSize;
 125     };
 126
 127     typedef SGSharedPtr<HTTPRepoGetRequest> RepoRequestPtr;
 128
 129 class HTTPRepoPrivate
 130 {
 131 public:
 132     struct HashCacheEntry
 133     {
 134         std::string filePath;
 135         time_t modTime;
 136         size_t lengthBytes;
 137         std::string hashHex;
 138
 139     };
 140
 141     typedef std::vector<HashCacheEntry> HashCache;
 142     HashCache hashes;
 143
 144     struct Failure
 145     {
 146         SGPath path;
 147         AbstractRepository::ResultCode error;
 148     };
 149
 150     typedef std::vector<Failure> FailureList;
 151     FailureList failures;
 152
 153     HTTPRepoPrivate(HTTPRepository* parent) :
 154     p(parent),
 155     isUpdating(false),
 156     status(AbstractRepository::REPO_NO_ERROR),
 157     totalDownloaded(0)
 158     { ; }
 159
 160     ~HTTPRepoPrivate();
 161
 162     HTTPRepository* p; // link back to outer
 163     HTTP::Client* http;
 164     std::string baseUrl;
 165     SGPath basePath;
 166     bool isUpdating;
 167     AbstractRepository::ResultCode status;
 168     HTTPDirectory* rootDir;
 169     size_t totalDownloaded;
 170
 171     HTTP::Request_ptr updateFile(HTTPDirectory* dir, const std::string& name,
 172                                  size_t sz);
 173     HTTP::Request_ptr updateDir(HTTPDirectory* dir, const std::string& hash,
 174                                 size_t sz);
 175
 176     std::string hashForPath(const SGPath& p);
 177     void updatedFileContents(const SGPath& p, const std::string& newHash);
 178     void parseHashCache();
 179     std::string computeHashForPath(const SGPath& p);
 180     void writeHashCache();
 181
 182     void failedToGetRootIndex(AbstractRepository::ResultCode st);
 183     void failedToUpdateChild(const SGPath& relativePath,
 184                              AbstractRepository::ResultCode fileStatus);
 185
 186     typedef std::vector<RepoRequestPtr> RequestVector;
 187     RequestVector requests;
 188
 189     void finishedRequest(const RepoRequestPtr& req);
 190
 191     HTTPDirectory* getOrCreateDirectory(const std::string& path);
 192     bool deleteDirectory(const std::string& path);
 193
 194     typedef std::vector<HTTPDirectory*> DirectoryVector;
 195     DirectoryVector directories;
 196
 197 };
 198
 199 class HTTPDirectory
 200 {
 201     struct ChildInfo
 202     {
 203         enum Type
 204         {
 205             FileType,
 206             DirectoryType
 207         };
 208
 209         ChildInfo(Type ty, const char* nameData, const char* hashData) :
 210             type(ty),
 211             name(nameData),
 212             hash(hashData ? hashData : ""),
 213             sizeInBytes(0)
 214         {
 215         }
 216
 217         ChildInfo(const ChildInfo& other) :
 218             type(other.type),
 219             name(other.name),
 220             hash(other.hash),
 221             sizeInBytes(other.sizeInBytes)
 222         { }
 223
 224         void setSize(const char* sizeData)
 225         {
 226             sizeInBytes = ::strtol(sizeData, NULL, 10);
 227         }
 228
 229         bool operator<(const ChildInfo& other) const
 230         {
 231             return name < other.name;
 232         }
 233
 234         Type type;
 235         std::string name, hash;
 236         size_t sizeInBytes;
 237     };
 238
 239     typedef std::vector<ChildInfo> ChildInfoList;
 240     ChildInfoList children;
 241
 242 public:
 243     HTTPDirectory(HTTPRepoPrivate* repo, const std::string& path) :
 244         _repository(repo),
 245         _relativePath(path)
 246   {
 247       assert(repo);
 248
 249       SGPath p(absolutePath());
 250       if (p.exists()) {
 251           try {
 252               // already exists on disk
 253               parseDirIndex(children);
 254               std::sort(children.begin(), children.end());
 255           } catch (sg_exception& e) {
 256               // parsing cache failed
 257               children.clear();
 258           }
 259       }
 260   }
 261
 262     HTTPRepoPrivate* repository() const
 263     {
 264         return _repository;
 265     }
 266
 267     std::string url() const
 268     {
 269         if (_relativePath.str().empty()) {
 270             return _repository->baseUrl;
 271         }
 272
 273         return _repository->baseUrl + "/" + _relativePath.str();
 274     }
 275
 276     void dirIndexUpdated(const std::string& hash)
 277     {
 278         SGPath fpath(_relativePath);
 279         fpath.append(".dirindex");
 280         _repository->updatedFileContents(fpath, hash);
 281
 282         children.clear();
 283         parseDirIndex(children);
 284         std::sort(children.begin(), children.end());
 285     }
 286
 287     void failedToUpdate(AbstractRepository::ResultCode status)
 288     {
 289         if (_relativePath.isNull()) {
 290             // root dir failed
 291             _repository->failedToGetRootIndex(status);
 292         } else {
 293             _repository->failedToUpdateChild(_relativePath, status);
 294         }
 295     }
 296
 297     void updateChildrenBasedOnHash()
 298     {
 299         //SG_LOG(SG_TERRASYNC, SG_DEBUG, "updated children for:" << relativePath());
 300
 301         string_list indexNames = indexChildren(),
 302             toBeUpdated, orphans;
 303         simgear::Dir d(absolutePath());
 304         PathList fsChildren = d.children(0);
 305         PathList::const_iterator it = fsChildren.begin();
 306
 307         for (; it != fsChildren.end(); ++it) {
 308             ChildInfo info(it->isDir() ? ChildInfo::DirectoryType : ChildInfo::FileType,
 309                            it->file().c_str(), NULL);
 310             std::string hash = hashForChild(info);
 311
 312             ChildInfoList::iterator c = findIndexChild(it->file());
 313             if (c == children.end()) {
 314                 orphans.push_back(it->file());
 315             } else if (c->hash != hash) {
 316                 // file exists, but hash mismatch, schedule update
 317                 if (!hash.empty()) {
 318                     //SG_LOG(SG_TERRASYNC, SG_INFO, "file exists but hash is wrong for:" << c->name);
 319                     //SG_LOG(SG_TERRASYNC, SG_INFO, "on disk:" << hash << " vs in info:" << c->hash);
 320                 }
 321
 322                 toBeUpdated.push_back(c->name);
 323             } else {
 324                 // file exists and hash is valid. If it's a directory,
 325                 // perform a recursive check.
 326                 if (c->type == ChildInfo::DirectoryType) {
 327                     SGPath p(relativePath());
 328                     p.append(c->name);
 329                     HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str());
 330                     childDir->updateChildrenBasedOnHash();
 331                 }
 332             }
 333
 334             // remove existing file system children from the index list,
 335             // so we can detect new children
 336             string_list::iterator it = std::find(indexNames.begin(), indexNames.end(), c->name);
 337             if (it != indexNames.end()) {
 338                 indexNames.erase(it);
 339             }
 340         } // of real children iteration
 341
 342         // all remaining names in indexChilden are new children
 343         toBeUpdated.insert(toBeUpdated.end(), indexNames.begin(), indexNames.end());
 344
 345         removeOrphans(orphans);
 346         scheduleUpdates(toBeUpdated);
 347     }
 348
 349     void removeOrphans(const string_list& orphans)
 350     {
 351         string_list::const_iterator it;
 352         for (it = orphans.begin(); it != orphans.end(); ++it) {
 353             removeChild(*it);
 354         }
 355     }
 356
 357     string_list indexChildren() const
 358     {
 359         string_list r;
 360         r.reserve(children.size());
 361         ChildInfoList::const_iterator it;
 362         for (it=children.begin(); it != children.end(); ++it) {
 363             r.push_back(it->name);
 364         }
 365         return r;
 366     }
 367
 368     void scheduleUpdates(const string_list& names)
 369     {
 370         string_list::const_iterator it;
 371         for (it = names.begin(); it != names.end(); ++it) {
 372             ChildInfoList::iterator cit = findIndexChild(*it);
 373             if (cit == children.end()) {
 374                 SG_LOG(SG_TERRASYNC, SG_WARN, "scheduleUpdate, unknown child:" << *it);
 375                 continue;
 376             }
 377
 378             if (cit->type == ChildInfo::FileType) {
 379                 _repository->updateFile(this, *it, cit->sizeInBytes);
 380             } else {
 381                 SGPath p(relativePath());
 382                 p.append(*it);
 383                 HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str());
 384                 _repository->updateDir(childDir, cit->hash, cit->sizeInBytes);
 385             }
 386         }
 387     }
 388
 389     SGPath absolutePath() const
 390     {
 391         SGPath r(_repository->basePath);
 392         r.append(_relativePath.str());
 393         return r;
 394     }
 395
 396     SGPath relativePath() const
 397     {
 398         return _relativePath;
 399     }
 400
 401     void didUpdateFile(const std::string& file, const std::string& hash, size_t sz)
 402     {
 403         // check hash matches what we expected
 404         ChildInfoList::iterator it = findIndexChild(file);
 405         if (it == children.end()) {
 406             SG_LOG(SG_TERRASYNC, SG_WARN, "updated file but not found in dir:" << _relativePath << " " << file);
 407         } else {
 408             SGPath fpath(_relativePath);
 409             fpath.append(file);
 410
 411             if (it->hash != hash) {
 412                 _repository->failedToUpdateChild(_relativePath, AbstractRepository::REPO_ERROR_CHECKSUM);
 413             } else {
 414                 _repository->updatedFileContents(fpath, hash);
 415                 _repository->totalDownloaded += sz;
 416                 //SG_LOG(SG_TERRASYNC, SG_INFO, "did update:" << fpath);
 417             } // of hash matches
 418         } // of found in child list
 419     }
 420
 421     void didFailToUpdateFile(const std::string& file,
 422                              AbstractRepository::ResultCode status)
 423     {
 424         SGPath fpath(_relativePath);
 425         fpath.append(file);
 426         _repository->failedToUpdateChild(fpath, status);
 427     }
 428 private:
 429
 430     struct ChildWithName
 431     {
 432         ChildWithName(const std::string& n) : name(n) {}
 433         std::string name;
 434
 435         bool operator()(const ChildInfo& info) const
 436         { return info.name == name; }
 437     };
 438
 439     ChildInfoList::iterator findIndexChild(const std::string& name)
 440     {
 441         return std::find_if(children.begin(), children.end(), ChildWithName(name));
 442     }
 443
 444     bool parseDirIndex(ChildInfoList& children)
 445     {
 446         SGPath p(absolutePath());
 447         p.append(".dirindex");
 448         if (!p.exists()) {
 449             return false;
 450         }
 451
 452         std::ifstream indexStream( p.c_str(), std::ios::in );
 453
 454         if ( !indexStream.is_open() ) {
 455             throw sg_io_exception("cannot open dirIndex file", p);
 456         }
 457
 458         char lineBuffer[512];
 459         char* lastToken;
 460
 461         while (!indexStream.eof() ) {
 462             indexStream.getline(lineBuffer, 512);
 463             lastToken = 0;
 464             char* typeData = ::strtok_r(lineBuffer, ":", &lastToken);
 465             if (!typeData) {
 466                 continue; // skip blank line
 467             }
 468
 469             if (!typeData) {
 470                 // malformed entry
 471                 throw sg_io_exception("Malformed dir index file", p);
 472             }
 473
 474             if (!strcmp(typeData, "version")) {
 475                 continue;
 476             } else if (!strcmp(typeData, "path")) {
 477                 continue;
 478             }
 479
 480             char* nameData = ::strtok_r(NULL, ":", &lastToken);
 481             char* hashData = ::strtok_r(NULL, ":", &lastToken);
 482             char* sizeData = ::strtok_r(NULL, ":", &lastToken);
 483
 484             if (typeData[0] == 'f') {
 485                 children.push_back(ChildInfo(ChildInfo::FileType, nameData, hashData));
 486             } else if (typeData[0] == 'd') {
 487                 children.push_back(ChildInfo(ChildInfo::DirectoryType, nameData, hashData));
 488             } else {
 489                 throw sg_io_exception("Malformed line code in dir index file", p);
 490             }
 491
 492             if (sizeData) {
 493                 children.back().setSize(sizeData);
 494             }
 495         }
 496
 497         return true;
 498     }
 499
 500     void removeChild(const std::string& name)
 501     {
 502         SGPath p(absolutePath());
 503         p.append(name);
 504         bool ok;
 505
 506         SGPath fpath(_relativePath);
 507         fpath.append(name);
 508
 509         if (p.isDir()) {
 510             ok = _repository->deleteDirectory(fpath.str());
 511         } else {
 512             // remove the hash cache entry
 513             _repository->updatedFileContents(fpath, std::string());
 514             ok = p.remove();
 515         }
 516
 517         if (!ok) {
 518             SG_LOG(SG_TERRASYNC, SG_WARN, "removal failed for:" << p);
 519             throw sg_io_exception("Failed to remove existing file/dir:", p);
 520         }
 521     }
 522
 523     std::string hashForChild(const ChildInfo& child) const
 524     {
 525         SGPath p(absolutePath());
 526         p.append(child.name);
 527         if (child.type == ChildInfo::DirectoryType) {
 528             p.append(".dirindex");
 529         }
 530         return _repository->hashForPath(p);
 531     }
 532
 533   HTTPRepoPrivate* _repository;
 534   SGPath _relativePath; // in URL and file-system space
 535
 536
 537 };
 538
 539 HTTPRepository::HTTPRepository(const SGPath& base, HTTP::Client *cl) :
 540     _d(new HTTPRepoPrivate(this))
 541 {
 542     _d->http = cl;
 543     _d->basePath = base;
 544     _d->rootDir = new HTTPDirectory(_d.get(), "");
 545 }
 546
 547 HTTPRepository::~HTTPRepository()
 548 {
 549 }
 550
 551 void HTTPRepository::setBaseUrl(const std::string &url)
 552 {
 553   _d->baseUrl = url;
 554 }
 555
 556 std::string HTTPRepository::baseUrl() const
 557 {
 558   return _d->baseUrl;
 559 }
 560
 561 HTTP::Client* HTTPRepository::http() const
 562 {
 563   return _d->http;
 564 }
 565
 566 SGPath HTTPRepository::fsBase() const
 567 {
 568   return SGPath();
 569 }
 570
 571 void HTTPRepository::update()
 572 {
 573     if (_d->isUpdating) {
 574         return;
 575     }
 576
 577     _d->status = REPO_NO_ERROR;
 578     _d->isUpdating = true;
 579     _d->failures.clear();
 580     _d->updateDir(_d->rootDir, std::string(), 0);
 581 }
 582
 583 bool HTTPRepository::isDoingSync() const
 584 {
 585     if (_d->status != REPO_NO_ERROR) {
 586         return false;
 587     }
 588
 589     return _d->isUpdating;
 590 }
 591
 592 size_t HTTPRepository::bytesToDownload() const
 593 {
 594     size_t result = 0;
 595
 596     HTTPRepoPrivate::RequestVector::const_iterator r;
 597     for (r = _d->requests.begin(); r != _d->requests.end(); ++r) {
 598         result += (*r)->contentSize() - (*r)->responseBytesReceived();
 599     }
 600
 601     return result;
 602 }
 603
 604 size_t HTTPRepository::bytesDownloaded() const
 605 {
 606     size_t result = _d->totalDownloaded;
 607
 608     HTTPRepoPrivate::RequestVector::const_iterator r;
 609     for (r = _d->requests.begin(); r != _d->requests.end(); ++r) {
 610         result += (*r)->responseBytesReceived();
 611     }
 612
 613     return result;
 614 }
 615
 616 AbstractRepository::ResultCode
 617 HTTPRepository::failure() const
 618 {
 619     if ((_d->status == REPO_NO_ERROR) && !_d->failures.empty()) {
 620         return REPO_PARTIAL_UPDATE;
 621     }
 622
 623     return _d->status;
 624 }
 625
 626     class FileGetRequest : public HTTPRepoGetRequest
 627     {
 628     public:
 629         FileGetRequest(HTTPDirectory* d, const std::string& file) :
 630             HTTPRepoGetRequest(d, makeUrl(d, file)),
 631             fileName(file)
 632         {
 633             pathInRepo = _directory->absolutePath();
 634             pathInRepo.append(fileName);
 635             //SG_LOG(SG_TERRASYNC, SG_INFO, "will GET file " << url());
 636         }
 637
 638     protected:
 639         virtual void gotBodyData(const char* s, int n)
 640         {
 641             if (!file.get()) {
 642                 file.reset(new SGFile(pathInRepo.str()));
 643                 if (!file->open(SG_IO_OUT)) {
 644                   SG_LOG(SG_TERRASYNC, SG_WARN, "unable to create file " << pathInRepo);
 645                   abort("Unable to create output file");
 646                 }
 647
 648                 sha1_init(&hashContext);
 649             }
 650
 651             sha1_write(&hashContext, s, n);
 652             file->write(s, n);
 653         }
 654
 655         virtual void onDone()
 656         {
 657             file->close();
 658             if (responseCode() == 200) {
 659                 std::string hash = strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
 660                 _directory->didUpdateFile(fileName, hash, contentSize());
 661                 //SG_LOG(SG_TERRASYNC, SG_INFO, "got file " << fileName << " in " << _directory->absolutePath());
 662             } else if (responseCode() == 404) {
 663                 _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_FILE_NOT_FOUND);
 664             } else {
 665                 _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_HTTP);
 666             }
 667
 668             _directory->repository()->finishedRequest(this);
 669         }
 670
 671         virtual void onFail()
 672         {
 673             file.reset();
 674             if (pathInRepo.exists()) {
 675                 pathInRepo.remove();
 676             }
 677
 678             if (_directory) {
 679                 _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_SOCKET);
 680                 _directory->repository()->finishedRequest(this);
 681             }
 682         }
 683     private:
 684         static std::string makeUrl(HTTPDirectory* d, const std::string& file)
 685         {
 686             return d->url() + "/" + file;
 687         }
 688
 689         std::string fileName; // if empty, we're getting the directory itself
 690         SGPath pathInRepo;
 691         simgear::sha1nfo hashContext;
 692         std::auto_ptr<SGFile> file;
 693     };
 694
 695     class DirGetRequest : public HTTPRepoGetRequest
 696     {
 697     public:
 698         DirGetRequest(HTTPDirectory* d, const std::string& targetHash) :
 699             HTTPRepoGetRequest(d, makeUrl(d)),
 700             _isRootDir(false),
 701             _targetHash(targetHash)
 702         {
 703             sha1_init(&hashContext);
 704            //SG_LOG(SG_TERRASYNC, SG_INFO, "will GET dir " << url());
 705         }
 706
 707         void setIsRootDir()
 708         {
 709             _isRootDir = true;
 710         }
 711
 712         bool isRootDir() const
 713         {
 714             return _isRootDir;
 715         }
 716
 717     protected:
 718         virtual void gotBodyData(const char* s, int n)
 719         {
 720             body += std::string(s, n);
 721             sha1_write(&hashContext, s, n);
 722         }
 723
 724         virtual void onDone()
 725         {
 726             if (responseCode() == 200) {
 727                 std::string hash = strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
 728                 if (!_targetHash.empty() && (hash != _targetHash)) {
 729                     _directory->failedToUpdate(AbstractRepository::REPO_ERROR_CHECKSUM);
 730                     _directory->repository()->finishedRequest(this);
 731                     return;
 732                 }
 733
 734                 std::string curHash = _directory->repository()->hashForPath(path());
 735                 if (hash != curHash) {
 736                     simgear::Dir d(_directory->absolutePath());
 737                     if (!d.exists()) {
 738                         if (!d.create(0700)) {
 739                             throw sg_io_exception("Unable to create directory", d.path());
 740                         }
 741                     }
 742
 743                     // dir index data has changed, so write to disk and update
 744                     // the hash accordingly
 745                     std::ofstream of(pathInRepo().c_str(), std::ios::trunc | std::ios::out);
 746                     if (!of.is_open()) {
 747                         throw sg_io_exception("Failed to open directory index file for writing", pathInRepo().c_str());
 748                     }
 749
 750                     of.write(body.data(), body.size());
 751                     of.close();
 752                     _directory->dirIndexUpdated(hash);
 753
 754                     //SG_LOG(SG_TERRASYNC, SG_INFO, "updated dir index " << _directory->absolutePath());
 755                 }
 756
 757                 _directory->repository()->totalDownloaded += contentSize();
 758
 759                 try {
 760                     // either way we've confirmed the index is valid so update
 761                     // children now
 762                     _directory->updateChildrenBasedOnHash();
 763                 } catch (sg_exception& e) {
 764                     _directory->failedToUpdate(AbstractRepository::REPO_ERROR_IO);
 765                 }
 766             } else if (responseCode() == 404) {
 767                 _directory->failedToUpdate(AbstractRepository::REPO_ERROR_FILE_NOT_FOUND);
 768             } else {
 769                 _directory->failedToUpdate(AbstractRepository::REPO_ERROR_HTTP);
 770             }
 771
 772             _directory->repository()->finishedRequest(this);
 773         }
 774
 775         virtual void onFail()
 776         {
 777             if (_directory) {
 778                 _directory->failedToUpdate(AbstractRepository::REPO_ERROR_SOCKET);
 779                 _directory->repository()->finishedRequest(this);
 780             }
 781         }
 782     private:
 783         static std::string makeUrl(HTTPDirectory* d)
 784         {
 785             return d->url() + "/.dirindex";
 786         }
 787
 788         SGPath pathInRepo() const
 789         {
 790             SGPath p(_directory->absolutePath());
 791             p.append(".dirindex");
 792             return p;
 793         }
 794
 795         simgear::sha1nfo hashContext;
 796         std::string body;
 797         bool _isRootDir; ///< is this the repository root?
 798         std::string _targetHash;
 799     };
 800
 801     HTTPRepoPrivate::~HTTPRepoPrivate()
 802     {
 803         DirectoryVector::iterator it;
 804         for (it=directories.begin(); it != directories.end(); ++it) {
 805             delete *it;
 806         }
 807
 808         RequestVector::iterator r;
 809         for (r=requests.begin(); r != requests.end(); ++r) {
 810             (*r)->cancel();
 811         }
 812     }
 813
 814     HTTP::Request_ptr HTTPRepoPrivate::updateFile(HTTPDirectory* dir, const std::string& name, size_t sz)
 815     {
 816         RepoRequestPtr r(new FileGetRequest(dir, name));
 817         r->setContentSize(sz);
 818         requests.push_back(r);
 819         http->makeRequest(r);
 820         return r;
 821     }
 822
 823     HTTP::Request_ptr HTTPRepoPrivate::updateDir(HTTPDirectory* dir, const std::string& hash, size_t sz)
 824     {
 825         RepoRequestPtr r(new DirGetRequest(dir, hash));
 826         r->setContentSize(sz);
 827         requests.push_back(r);
 828         http->makeRequest(r);
 829         return r;
 830     }
 831
 832
 833     class HashEntryWithPath
 834     {
 835     public:
 836         HashEntryWithPath(const std::string& p) : path(p) {}
 837         bool operator()(const HTTPRepoPrivate::HashCacheEntry& entry) const
 838         { return entry.filePath == path; }
 839     private:
 840         std::string path;
 841     };
 842
 843     std::string HTTPRepoPrivate::hashForPath(const SGPath& p)
 844     {
 845         HashCache::iterator it = std::find_if(hashes.begin(), hashes.end(), HashEntryWithPath(p.str()));
 846         if (it != hashes.end()) {
 847             // ensure data on disk hasn't changed.
 848             // we could also use the file type here if we were paranoid
 849             if ((p.sizeInBytes() == it->lengthBytes) && (p.modTime() == it->modTime)) {
 850                 return it->hashHex;
 851             }
 852
 853             // entry in the cache, but it's stale so remove and fall through
 854             hashes.erase(it);
 855         }
 856
 857         std::string hash = computeHashForPath(p);
 858         updatedFileContents(p, hash);
 859         return hash;
 860     }
 861
 862     std::string HTTPRepoPrivate::computeHashForPath(const SGPath& p)
 863     {
 864         if (!p.exists())
 865             return std::string();
 866         sha1nfo info;
 867         sha1_init(&info);
 868         char* buf = static_cast<char*>(malloc(1024 * 1024));
 869         size_t readLen;
 870         SGFile f(p.str());
 871         if (!f.open(SG_IO_IN)) {
 872             throw sg_io_exception("Couldn't open file for compute hash", p);
 873         }
 874         while ((readLen = f.read(buf, 1024 * 1024)) > 0) {
 875             sha1_write(&info, buf, readLen);
 876         }
 877
 878         f.close();
 879         free(buf);
 880         std::string hashBytes((char*) sha1_result(&info), HASH_LENGTH);
 881         return strutils::encodeHex(hashBytes);
 882     }
 883
 884     void HTTPRepoPrivate::updatedFileContents(const SGPath& p, const std::string& newHash)
 885     {
 886         // remove the existing entry
 887         HashCache::iterator it = std::find_if(hashes.begin(), hashes.end(), HashEntryWithPath(p.str()));
 888         if (it != hashes.end()) {
 889             hashes.erase(it);
 890         }
 891
 892         if (newHash.empty()) {
 893             return; // we're done
 894         }
 895
 896         // use a cloned SGPath and reset its caching to force one stat() call
 897         SGPath p2(p);
 898         p2.set_cached(false);
 899         p2.set_cached(true);
 900
 901         HashCacheEntry entry;
 902         entry.filePath = p.str();
 903         entry.hashHex = newHash;
 904         entry.modTime = p2.modTime();
 905         entry.lengthBytes = p2.sizeInBytes();
 906         hashes.push_back(entry);
 907
 908         writeHashCache();
 909     }
 910
 911     void HTTPRepoPrivate::writeHashCache()
 912     {
 913         SGPath cachePath = basePath;
 914         cachePath.append(".hashes");
 915
 916         std::ofstream stream(cachePath.c_str(),std::ios::out | std::ios::trunc);
 917         HashCache::const_iterator it;
 918         for (it = hashes.begin(); it != hashes.end(); ++it) {
 919             stream << it->filePath << ":" << it->modTime << ":"
 920             << it->lengthBytes << ":" << it->hashHex << "\n";
 921         }
 922         stream.close();
 923     }
 924
 925     void HTTPRepoPrivate::parseHashCache()
 926     {
 927         hashes.clear();
 928         SGPath cachePath = basePath;
 929         cachePath.append(".hashes");
 930         if (!cachePath.exists()) {
 931             return;
 932         }
 933
 934         std::ifstream stream(cachePath.c_str(), std::ios::in);
 935         char buf[2048];
 936         char* lastToken;
 937
 938         while (!stream.eof()) {
 939             stream.getline(buf, 2048);
 940             lastToken = 0;
 941             char* nameData = ::strtok_r(buf, ":", &lastToken);
 942             char* timeData = ::strtok_r(NULL, ":", &lastToken);
 943             char* sizeData = ::strtok_r(NULL, ":", &lastToken);
 944             char* hashData = ::strtok_r(NULL, ":", &lastToken);
 945             if (!nameData || !timeData || !sizeData || !hashData) {
 946                 continue;
 947             }
 948
 949             HashCacheEntry entry;
 950             entry.filePath = nameData;
 951             entry.hashHex = hashData;
 952             entry.modTime = strtol(timeData, NULL, 10);
 953             entry.lengthBytes = strtol(sizeData, NULL, 10);
 954             hashes.push_back(entry);
 955         }
 956     }
 957
 958     class DirectoryWithPath
 959     {
 960     public:
 961         DirectoryWithPath(const std::string& p) : path(p) {}
 962         bool operator()(const HTTPDirectory* entry) const
 963         { return entry->relativePath().str() == path; }
 964     private:
 965         std::string path;
 966     };
 967
 968     HTTPDirectory* HTTPRepoPrivate::getOrCreateDirectory(const std::string& path)
 969     {
 970         DirectoryWithPath p(path);
 971         DirectoryVector::iterator it = std::find_if(directories.begin(), directories.end(), p);
 972         if (it != directories.end()) {
 973             return *it;
 974         }
 975
 976         HTTPDirectory* d = new HTTPDirectory(this, path);
 977         directories.push_back(d);
 978         return d;
 979     }
 980
 981     bool HTTPRepoPrivate::deleteDirectory(const std::string& path)
 982     {
 983         DirectoryWithPath p(path);
 984         DirectoryVector::iterator it = std::find_if(directories.begin(), directories.end(), p);
 985         if (it != directories.end()) {
 986             HTTPDirectory* d = *it;
 987             directories.erase(it);
 988             Dir dir(d->absolutePath());
 989             bool result = dir.remove(true);
 990             delete d;
 991
 992             // update the hash cache too
 993             updatedFileContents(path, std::string());
 994
 995             return result;
 996         }
 997
 998         return false;
 999     }
1000
1001     void HTTPRepoPrivate::finishedRequest(const RepoRequestPtr& req)
1002     {
1003         RequestVector::iterator it = std::find(requests.begin(), requests.end(), req);
1004         if (it == requests.end()) {
1005             throw sg_exception("lost request somehow", req->url());
1006         }
1007         requests.erase(it);
1008         if (requests.empty()) {
1009             isUpdating = false;
1010         }
1011     }
1012
1013     void HTTPRepoPrivate::failedToGetRootIndex(AbstractRepository::ResultCode st)
1014     {
1015         SG_LOG(SG_TERRASYNC, SG_WARN, "Failed to get root of repo:" << baseUrl);
1016         status = st;
1017     }
1018
1019     void HTTPRepoPrivate::failedToUpdateChild(const SGPath& relativePath,
1020                                               AbstractRepository::ResultCode fileStatus)
1021     {
1022         Failure f;
1023         f.path = relativePath;
1024         f.error = fileStatus;
1025         failures.push_back(f);
1026
1027         SG_LOG(SG_TERRASYNC, SG_WARN, "failed to update entry:" << relativePath << " code:" << fileStatus);
1028     }
1029
1030
1031
1032 } // of namespace simgear