simgear/io/HTTPRepository.cxx

   1 // HTTPRepository.cxx -- plain HTTP TerraSync remote client
   2 //
   3 // Copyright (C) 20126  James Turner <zakalawe@mac.com>
   4 //
   5 // This program is free software; you can redistribute it and/or
   6 // modify it under the terms of the GNU General Public License as
   7 // published by the Free Software Foundation; either version 2 of the
   8 // License, or (at your option) any later version.
   9 //
  10 // This program is distributed in the hope that it will be useful, but
  11 // WITHOUT ANY WARRANTY; without even the implied warranty of
  12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 // General Public License for more details.
  14 //
  15 // You should have received a copy of the GNU General Public License
  16 // along with this program; if not, write to the Free Software
  17 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  18
  19 #include "HTTPRepository.hxx"
  20
  21 #include <simgear_config.h>
  22
  23 #include <iostream>
  24 #include <cstring>
  25 #include <cassert>
  26 #include <algorithm>
  27 #include <sstream>
  28 #include <map>
  29 #include <set>
  30 #include <fstream>
  31 #include <limits>
  32 #include <cstdlib>
  33
  34 #include <fcntl.h>
  35
  36 #include "simgear/debug/logstream.hxx"
  37 #include "simgear/misc/strutils.hxx"
  38 #include <simgear/misc/sg_dir.hxx>
  39 #include <simgear/io/HTTPClient.hxx>
  40 #include <simgear/io/sg_file.hxx>
  41 #include <simgear/misc/sgstream.hxx>
  42 #include <simgear/structure/exception.hxx>
  43
  44 #include <simgear/misc/sg_hash.hxx>
  45
  46 #if defined(SG_WINDOWS)
  47
  48 /*
  49  * public domain strtok_r() by Charlie Gordon
  50  *
  51  *   from comp.lang.c  9/14/2007
  52  *
  53  *      http://groups.google.com/group/comp.lang.c/msg/2ab1ecbb86646684
  54  *
  55  *     (Declaration that it's public domain):
  56  *      http://groups.google.com/group/comp.lang.c/msg/7c7b39328fefab9c
  57  */
  58
  59 char* strtok_r(
  60     char *str,
  61     const char *delim,
  62     char **nextp)
  63 {
  64     char *ret;
  65
  66     if (str == NULL)
  67     {
  68         str = *nextp;
  69     }
  70
  71     str += strspn(str, delim);
  72
  73     if (*str == '\0')
  74     {
  75         return NULL;
  76     }
  77
  78     ret = str;
  79
  80     str += strcspn(str, delim);
  81
  82     if (*str)
  83     {
  84         *str++ = '\0';
  85     }
  86
  87     *nextp = str;
  88
  89     return ret;
  90 }
  91 #endif
  92
  93 namespace simgear
  94 {
  95
  96     class HTTPDirectory;
  97
  98     class HTTPRepoGetRequest : public HTTP::Request
  99     {
 100     public:
 101         HTTPRepoGetRequest(HTTPDirectory* d, const std::string& u) :
 102             HTTP::Request(u),
 103             _directory(d)
 104         {
 105         }
 106
 107         virtual void cancel()
 108         {
 109             _directory = 0;
 110             abort("Repository cancelled request");
 111         }
 112     protected:
 113         HTTPDirectory* _directory;
 114     };
 115
 116     typedef SGSharedPtr<HTTPRepoGetRequest> RepoRequestPtr;
 117
 118 class HTTPRepoPrivate
 119 {
 120 public:
 121     struct HashCacheEntry
 122     {
 123         std::string filePath;
 124         time_t modTime;
 125         size_t lengthBytes;
 126         std::string hashHex;
 127
 128     };
 129
 130     typedef std::vector<HashCacheEntry> HashCache;
 131     HashCache hashes;
 132
 133     struct Failure
 134     {
 135         SGPath path;
 136         AbstractRepository::ResultCode error;
 137     };
 138
 139     typedef std::vector<Failure> FailureList;
 140     FailureList failures;
 141
 142     HTTPRepoPrivate(HTTPRepository* parent) :
 143     p(parent),
 144     isUpdating(false),
 145     status(AbstractRepository::REPO_NO_ERROR)
 146     { ; }
 147
 148     ~HTTPRepoPrivate();
 149
 150     HTTPRepository* p; // link back to outer
 151     HTTP::Client* http;
 152     std::string baseUrl;
 153     SGPath basePath;
 154     bool isUpdating;
 155     AbstractRepository::ResultCode status;
 156     HTTPDirectory* rootDir;
 157
 158     HTTP::Request_ptr updateFile(HTTPDirectory* dir, const std::string& name);
 159     HTTP::Request_ptr updateDir(HTTPDirectory* dir, const std::string& hash);
 160
 161     std::string hashForPath(const SGPath& p);
 162     void updatedFileContents(const SGPath& p, const std::string& newHash);
 163     void parseHashCache();
 164     std::string computeHashForPath(const SGPath& p);
 165     void writeHashCache();
 166
 167     void failedToGetRootIndex(AbstractRepository::ResultCode st);
 168     void failedToUpdateChild(const SGPath& relativePath,
 169                              AbstractRepository::ResultCode fileStatus);
 170
 171     typedef std::vector<RepoRequestPtr> RequestVector;
 172     RequestVector requests;
 173
 174     void finishedRequest(const RepoRequestPtr& req);
 175
 176     HTTPDirectory* getOrCreateDirectory(const std::string& path);
 177     bool deleteDirectory(const std::string& path);
 178
 179     typedef std::vector<HTTPDirectory*> DirectoryVector;
 180     DirectoryVector directories;
 181
 182 };
 183
 184 class HTTPDirectory
 185 {
 186     struct ChildInfo
 187     {
 188         enum Type
 189         {
 190             FileType,
 191             DirectoryType
 192         };
 193
 194         ChildInfo(Type ty, const char* nameData, const char* hashData) :
 195             type(ty),
 196             name(nameData),
 197             hash(hashData ? hashData : ""),
 198             sizeInBytes(0)
 199         {
 200         }
 201
 202         ChildInfo(const ChildInfo& other) :
 203             type(other.type),
 204             name(other.name),
 205             hash(other.hash),
 206             sizeInBytes(other.sizeInBytes)
 207         { }
 208
 209         void setSize(const char* sizeData)
 210         {
 211             sizeInBytes = ::strtol(sizeData, NULL, 10);
 212         }
 213
 214         bool operator<(const ChildInfo& other) const
 215         {
 216             return name < other.name;
 217         }
 218
 219         Type type;
 220         std::string name, hash;
 221         size_t sizeInBytes;
 222     };
 223
 224     typedef std::vector<ChildInfo> ChildInfoList;
 225     ChildInfoList children;
 226
 227 public:
 228     HTTPDirectory(HTTPRepoPrivate* repo, const std::string& path) :
 229         _repository(repo),
 230         _relativePath(path)
 231   {
 232       assert(repo);
 233
 234       SGPath p(absolutePath());
 235       if (p.exists()) {
 236           try {
 237               // already exists on disk
 238               bool ok = parseDirIndex(children);
 239               std::sort(children.begin(), children.end());
 240           } catch (sg_exception& e) {
 241               // parsing cache failed
 242               children.clear();
 243           }
 244       }
 245   }
 246
 247     HTTPRepoPrivate* repository() const
 248     {
 249         return _repository;
 250     }
 251
 252     std::string url() const
 253     {
 254         if (_relativePath.str().empty()) {
 255             return _repository->baseUrl;
 256         }
 257
 258         return _repository->baseUrl + "/" + _relativePath.str();
 259     }
 260
 261     void dirIndexUpdated(const std::string& hash)
 262     {
 263         SGPath fpath(_relativePath);
 264         fpath.append(".dirindex");
 265         _repository->updatedFileContents(fpath, hash);
 266
 267         children.clear();
 268         parseDirIndex(children);
 269         std::sort(children.begin(), children.end());
 270     }
 271
 272     void failedToUpdate(AbstractRepository::ResultCode status)
 273     {
 274         if (_relativePath.isNull()) {
 275             // root dir failed
 276             _repository->failedToGetRootIndex(status);
 277         } else {
 278             _repository->failedToUpdateChild(_relativePath, status);
 279         }
 280     }
 281
 282     void updateChildrenBasedOnHash()
 283     {
 284         //SG_LOG(SG_TERRASYNC, SG_DEBUG, "updated children for:" << relativePath());
 285
 286         string_list indexNames = indexChildren(),
 287             toBeUpdated, orphans;
 288         simgear::Dir d(absolutePath());
 289         PathList fsChildren = d.children(0);
 290         PathList::const_iterator it = fsChildren.begin();
 291
 292         for (; it != fsChildren.end(); ++it) {
 293             ChildInfo info(it->isDir() ? ChildInfo::DirectoryType : ChildInfo::FileType,
 294                            it->file().c_str(), NULL);
 295             std::string hash = hashForChild(info);
 296
 297             ChildInfoList::iterator c = findIndexChild(it->file());
 298             if (c == children.end()) {
 299                 orphans.push_back(it->file());
 300             } else if (c->hash != hash) {
 301                 // file exists, but hash mismatch, schedule update
 302                 if (!hash.empty()) {
 303                     //SG_LOG(SG_TERRASYNC, SG_INFO, "file exists but hash is wrong for:" << c->name);
 304                     //SG_LOG(SG_TERRASYNC, SG_INFO, "on disk:" << hash << " vs in info:" << c->hash);
 305                 }
 306
 307                 toBeUpdated.push_back(c->name);
 308             } else {
 309                 // file exists and hash is valid. If it's a directory,
 310                 // perform a recursive check.
 311                 if (c->type == ChildInfo::DirectoryType) {
 312                     SGPath p(relativePath());
 313                     p.append(c->name);
 314                     HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str());
 315                     childDir->updateChildrenBasedOnHash();
 316                 }
 317             }
 318
 319             // remove existing file system children from the index list,
 320             // so we can detect new children
 321             string_list::iterator it = std::find(indexNames.begin(), indexNames.end(), c->name);
 322             if (it != indexNames.end()) {
 323                 indexNames.erase(it);
 324             }
 325         } // of real children iteration
 326
 327         // all remaining names in indexChilden are new children
 328         toBeUpdated.insert(toBeUpdated.end(), indexNames.begin(), indexNames.end());
 329
 330         removeOrphans(orphans);
 331         scheduleUpdates(toBeUpdated);
 332     }
 333
 334     void removeOrphans(const string_list& orphans)
 335     {
 336         string_list::const_iterator it;
 337         for (it = orphans.begin(); it != orphans.end(); ++it) {
 338             removeChild(*it);
 339         }
 340     }
 341
 342     string_list indexChildren() const
 343     {
 344         string_list r;
 345         r.reserve(children.size());
 346         ChildInfoList::const_iterator it;
 347         for (it=children.begin(); it != children.end(); ++it) {
 348             r.push_back(it->name);
 349         }
 350         return r;
 351     }
 352
 353     void scheduleUpdates(const string_list& names)
 354     {
 355         string_list::const_iterator it;
 356         for (it = names.begin(); it != names.end(); ++it) {
 357             ChildInfoList::iterator cit = findIndexChild(*it);
 358             if (cit == children.end()) {
 359                 SG_LOG(SG_TERRASYNC, SG_WARN, "scheduleUpdate, unknown child:" << *it);
 360                 continue;
 361             }
 362
 363             if (cit->type == ChildInfo::FileType) {
 364                 _repository->updateFile(this, *it);
 365             } else {
 366                 SGPath p(relativePath());
 367                 p.append(*it);
 368                 HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str());
 369                 _repository->updateDir(childDir, cit->hash);
 370             }
 371         }
 372     }
 373
 374     SGPath absolutePath() const
 375     {
 376         SGPath r(_repository->basePath);
 377         r.append(_relativePath.str());
 378         return r;
 379     }
 380
 381     SGPath relativePath() const
 382     {
 383         return _relativePath;
 384     }
 385
 386     void didUpdateFile(const std::string& file, const std::string& hash)
 387     {
 388         // check hash matches what we expected
 389         ChildInfoList::iterator it = findIndexChild(file);
 390         if (it == children.end()) {
 391             SG_LOG(SG_TERRASYNC, SG_WARN, "updated file but not found in dir:" << _relativePath << " " << file);
 392         } else {
 393             SGPath fpath(_relativePath);
 394             fpath.append(file);
 395
 396             if (it->hash != hash) {
 397                 _repository->failedToUpdateChild(_relativePath, AbstractRepository::REPO_ERROR_CHECKSUM);
 398             } else {
 399                 _repository->updatedFileContents(fpath, hash);
 400                 //SG_LOG(SG_TERRASYNC, SG_INFO, "did update:" << fpath);
 401             } // of hash matches
 402         } // of found in child list
 403     }
 404
 405     void didFailToUpdateFile(const std::string& file,
 406                              AbstractRepository::ResultCode status)
 407     {
 408         SGPath fpath(_relativePath);
 409         fpath.append(file);
 410         _repository->failedToUpdateChild(fpath, status);
 411     }
 412 private:
 413
 414     struct ChildWithName
 415     {
 416         ChildWithName(const std::string& n) : name(n) {}
 417         std::string name;
 418
 419         bool operator()(const ChildInfo& info) const
 420         { return info.name == name; }
 421     };
 422
 423     ChildInfoList::iterator findIndexChild(const std::string& name)
 424     {
 425         return std::find_if(children.begin(), children.end(), ChildWithName(name));
 426     }
 427
 428     bool parseDirIndex(ChildInfoList& children)
 429     {
 430         SGPath p(absolutePath());
 431         p.append(".dirindex");
 432         if (!p.exists()) {
 433             return false;
 434         }
 435
 436         std::ifstream indexStream( p.c_str(), std::ios::in );
 437
 438         if ( !indexStream.is_open() ) {
 439             throw sg_io_exception("cannot open dirIndex file", p);
 440         }
 441
 442         char lineBuffer[512];
 443         char* lastToken;
 444
 445         while (!indexStream.eof() ) {
 446             indexStream.getline(lineBuffer, 512);
 447             lastToken = 0;
 448             char* typeData = ::strtok_r(lineBuffer, ":", &lastToken);
 449             if (!typeData) {
 450                 continue; // skip blank line
 451             }
 452
 453             if (!typeData) {
 454                 // malformed entry
 455                 throw sg_io_exception("Malformed dir index file", p);
 456             }
 457
 458             if (!strcmp(typeData, "version")) {
 459                 continue;
 460             } else if (!strcmp(typeData, "path")) {
 461                 continue;
 462             }
 463
 464             char* nameData = ::strtok_r(NULL, ":", &lastToken);
 465             char* hashData = ::strtok_r(NULL, ":", &lastToken);
 466             char* sizeData = ::strtok_r(NULL, ":", &lastToken);
 467
 468             if (typeData[0] == 'f') {
 469                 children.push_back(ChildInfo(ChildInfo::FileType, nameData, hashData));
 470             } else if (typeData[0] == 'd') {
 471                 children.push_back(ChildInfo(ChildInfo::DirectoryType, nameData, hashData));
 472             } else {
 473                 throw sg_io_exception("Malformed line code in dir index file", p);
 474             }
 475
 476             if (sizeData) {
 477                 children.back().setSize(sizeData);
 478             }
 479         }
 480
 481         return true;
 482     }
 483
 484     void removeChild(const std::string& name)
 485     {
 486         SGPath p(absolutePath());
 487         p.append(name);
 488         bool ok;
 489
 490         SGPath fpath(_relativePath);
 491         fpath.append(name);
 492
 493         if (p.isDir()) {
 494             ok = _repository->deleteDirectory(fpath.str());
 495         } else {
 496             // remove the hash cache entry
 497             _repository->updatedFileContents(fpath, std::string());
 498             ok = p.remove();
 499         }
 500
 501         if (!ok) {
 502             SG_LOG(SG_TERRASYNC, SG_WARN, "removal failed for:" << p);
 503             throw sg_io_exception("Failed to remove existing file/dir:", p);
 504         }
 505     }
 506
 507     std::string hashForChild(const ChildInfo& child) const
 508     {
 509         SGPath p(absolutePath());
 510         p.append(child.name);
 511         if (child.type == ChildInfo::DirectoryType) {
 512             p.append(".dirindex");
 513         }
 514         return _repository->hashForPath(p);
 515     }
 516
 517   HTTPRepoPrivate* _repository;
 518   SGPath _relativePath; // in URL and file-system space
 519
 520
 521 };
 522
 523 HTTPRepository::HTTPRepository(const SGPath& base, HTTP::Client *cl) :
 524     _d(new HTTPRepoPrivate(this))
 525 {
 526     _d->http = cl;
 527     _d->basePath = base;
 528     _d->rootDir = new HTTPDirectory(_d.get(), "");
 529 }
 530
 531 HTTPRepository::~HTTPRepository()
 532 {
 533 }
 534
 535 void HTTPRepository::setBaseUrl(const std::string &url)
 536 {
 537   _d->baseUrl = url;
 538 }
 539
 540 std::string HTTPRepository::baseUrl() const
 541 {
 542   return _d->baseUrl;
 543 }
 544
 545 HTTP::Client* HTTPRepository::http() const
 546 {
 547   return _d->http;
 548 }
 549
 550 SGPath HTTPRepository::fsBase() const
 551 {
 552   return SGPath();
 553 }
 554
 555 void HTTPRepository::update()
 556 {
 557     if (_d->isUpdating) {
 558         return;
 559     }
 560
 561     _d->status = REPO_NO_ERROR;
 562     _d->isUpdating = true;
 563     _d->failures.clear();
 564     _d->updateDir(_d->rootDir, std::string());
 565 }
 566
 567 bool HTTPRepository::isDoingSync() const
 568 {
 569     if (_d->status != REPO_NO_ERROR) {
 570         return false;
 571     }
 572
 573     return _d->isUpdating;
 574 }
 575
 576 AbstractRepository::ResultCode
 577 HTTPRepository::failure() const
 578 {
 579     if ((_d->status == REPO_NO_ERROR) && !_d->failures.empty()) {
 580         return REPO_PARTIAL_UPDATE;
 581     }
 582
 583     return _d->status;
 584 }
 585
 586     class FileGetRequest : public HTTPRepoGetRequest
 587     {
 588     public:
 589         FileGetRequest(HTTPDirectory* d, const std::string& file) :
 590             HTTPRepoGetRequest(d, makeUrl(d, file)),
 591             fileName(file)
 592         {
 593             pathInRepo = _directory->absolutePath();
 594             pathInRepo.append(fileName);
 595             //SG_LOG(SG_TERRASYNC, SG_INFO, "will GET file " << url());
 596         }
 597
 598     protected:
 599         virtual void gotBodyData(const char* s, int n)
 600         {
 601             if (!file.get()) {
 602                 file.reset(new SGFile(pathInRepo.str()));
 603                 if (!file->open(SG_IO_OUT)) {
 604                   SG_LOG(SG_TERRASYNC, SG_WARN, "unable to create file " << pathInRepo);
 605                   abort("Unable to create output file");
 606                 }
 607
 608                 sha1_init(&hashContext);
 609             }
 610
 611             sha1_write(&hashContext, s, n);
 612             file->write(s, n);
 613         }
 614
 615         virtual void onDone()
 616         {
 617             file->close();
 618             if (responseCode() == 200) {
 619                 std::string hash = strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
 620                 _directory->didUpdateFile(fileName, hash);
 621                 //SG_LOG(SG_TERRASYNC, SG_INFO, "got file " << fileName << " in " << _directory->absolutePath());
 622             } else if (responseCode() == 404) {
 623                 _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_FILE_NOT_FOUND);
 624             } else {
 625                 _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_HTTP);
 626             }
 627
 628             _directory->repository()->finishedRequest(this);
 629         }
 630
 631         virtual void onFail()
 632         {
 633             file.reset();
 634             pathInRepo.remove();
 635             if (_directory) {
 636                 _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_SOCKET);
 637                 _directory->repository()->finishedRequest(this);
 638             }
 639         }
 640     private:
 641         static std::string makeUrl(HTTPDirectory* d, const std::string& file)
 642         {
 643             return d->url() + "/" + file;
 644         }
 645
 646         std::string fileName; // if empty, we're getting the directory itself
 647         SGPath pathInRepo;
 648         simgear::sha1nfo hashContext;
 649         std::auto_ptr<SGFile> file;
 650     };
 651
 652     class DirGetRequest : public HTTPRepoGetRequest
 653     {
 654     public:
 655         DirGetRequest(HTTPDirectory* d, const std::string& targetHash) :
 656             HTTPRepoGetRequest(d, makeUrl(d)),
 657             _isRootDir(false),
 658             _targetHash(targetHash)
 659         {
 660             sha1_init(&hashContext);
 661            //SG_LOG(SG_TERRASYNC, SG_INFO, "will GET dir " << url());
 662         }
 663
 664         void setIsRootDir()
 665         {
 666             _isRootDir = true;
 667         }
 668
 669         bool isRootDir() const
 670         {
 671             return _isRootDir;
 672         }
 673
 674     protected:
 675         virtual void gotBodyData(const char* s, int n)
 676         {
 677             body += std::string(s, n);
 678             sha1_write(&hashContext, s, n);
 679         }
 680
 681         virtual void onDone()
 682         {
 683             if (responseCode() == 200) {
 684                 std::string hash = strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
 685                 if (!_targetHash.empty() && (hash != _targetHash)) {
 686                     _directory->failedToUpdate(AbstractRepository::REPO_ERROR_CHECKSUM);
 687                     _directory->repository()->finishedRequest(this);
 688                     return;
 689                 }
 690
 691                 std::string curHash = _directory->repository()->hashForPath(path());
 692                 if (hash != curHash) {
 693                     simgear::Dir d(_directory->absolutePath());
 694                     if (!d.exists()) {
 695                         if (!d.create(0700)) {
 696                             throw sg_io_exception("Unable to create directory", d.path());
 697                         }
 698                     }
 699
 700                     // dir index data has changed, so write to disk and update
 701                     // the hash accordingly
 702                     std::ofstream of(pathInRepo().c_str(), std::ios::trunc | std::ios::out);
 703                     if (!of.is_open()) {
 704                         throw sg_io_exception("Failed to open directory index file for writing", pathInRepo().c_str());
 705                     }
 706
 707                     of.write(body.data(), body.size());
 708                     of.close();
 709                     _directory->dirIndexUpdated(hash);
 710
 711                     //SG_LOG(SG_TERRASYNC, SG_INFO, "updated dir index " << _directory->absolutePath());
 712                 }
 713
 714                 try {
 715                     // either way we've confirmed the index is valid so update
 716                     // children now
 717                     _directory->updateChildrenBasedOnHash();
 718                 } catch (sg_exception& e) {
 719                     _directory->failedToUpdate(AbstractRepository::REPO_ERROR_IO);
 720                 }
 721             } else if (responseCode() == 404) {
 722                 _directory->failedToUpdate(AbstractRepository::REPO_ERROR_FILE_NOT_FOUND);
 723             } else {
 724                 _directory->failedToUpdate(AbstractRepository::REPO_ERROR_HTTP);
 725             }
 726
 727             _directory->repository()->finishedRequest(this);
 728         }
 729
 730         virtual void onFail()
 731         {
 732             if (_directory) {
 733                 _directory->failedToUpdate(AbstractRepository::REPO_ERROR_SOCKET);
 734                 _directory->repository()->finishedRequest(this);
 735             }
 736         }
 737     private:
 738         static std::string makeUrl(HTTPDirectory* d)
 739         {
 740             return d->url() + "/.dirindex";
 741         }
 742
 743         SGPath pathInRepo() const
 744         {
 745             SGPath p(_directory->absolutePath());
 746             p.append(".dirindex");
 747             return p;
 748         }
 749
 750         simgear::sha1nfo hashContext;
 751         std::string body;
 752         bool _isRootDir; ///< is this the repository root?
 753         std::string _targetHash;
 754     };
 755
 756     HTTPRepoPrivate::~HTTPRepoPrivate()
 757     {
 758         DirectoryVector::iterator it;
 759         for (it=directories.begin(); it != directories.end(); ++it) {
 760             delete *it;
 761         }
 762
 763         RequestVector::iterator r;
 764         for (r=requests.begin(); r != requests.end(); ++r) {
 765             (*r)->cancel();
 766         }
 767     }
 768
 769     HTTP::Request_ptr HTTPRepoPrivate::updateFile(HTTPDirectory* dir, const std::string& name)
 770     {
 771         RepoRequestPtr r(new FileGetRequest(dir, name));
 772         requests.push_back(r);
 773         http->makeRequest(r);
 774         return r;
 775     }
 776
 777     HTTP::Request_ptr HTTPRepoPrivate::updateDir(HTTPDirectory* dir, const std::string& hash)
 778     {
 779         RepoRequestPtr r(new DirGetRequest(dir, hash));
 780         requests.push_back(r);
 781         http->makeRequest(r);
 782         return r;
 783     }
 784
 785
 786     class HashEntryWithPath
 787     {
 788     public:
 789         HashEntryWithPath(const std::string& p) : path(p) {}
 790         bool operator()(const HTTPRepoPrivate::HashCacheEntry& entry) const
 791         { return entry.filePath == path; }
 792     private:
 793         std::string path;
 794     };
 795
 796     std::string HTTPRepoPrivate::hashForPath(const SGPath& p)
 797     {
 798         HashCache::iterator it = std::find_if(hashes.begin(), hashes.end(), HashEntryWithPath(p.str()));
 799         if (it != hashes.end()) {
 800             // ensure data on disk hasn't changed.
 801             // we could also use the file type here if we were paranoid
 802             if ((p.sizeInBytes() == it->lengthBytes) && (p.modTime() == it->modTime)) {
 803                 return it->hashHex;
 804             }
 805
 806             // entry in the cache, but it's stale so remove and fall through
 807             hashes.erase(it);
 808         }
 809
 810         std::string hash = computeHashForPath(p);
 811         updatedFileContents(p, hash);
 812         return hash;
 813     }
 814
 815     std::string HTTPRepoPrivate::computeHashForPath(const SGPath& p)
 816     {
 817         if (!p.exists())
 818             return std::string();
 819         sha1nfo info;
 820         sha1_init(&info);
 821         char* buf = static_cast<char*>(malloc(1024 * 1024));
 822         size_t readLen;
 823         SGFile f(p.str());
 824         if (!f.open(SG_IO_IN)) {
 825             throw sg_io_exception("Couldn't open file for compute hash", p);
 826         }
 827         while ((readLen = f.read(buf, 1024 * 1024)) > 0) {
 828             sha1_write(&info, buf, readLen);
 829         }
 830
 831         f.close();
 832         free(buf);
 833         std::string hashBytes((char*) sha1_result(&info), HASH_LENGTH);
 834         return strutils::encodeHex(hashBytes);
 835     }
 836
 837     void HTTPRepoPrivate::updatedFileContents(const SGPath& p, const std::string& newHash)
 838     {
 839         // remove the existing entry
 840         HashCache::iterator it = std::find_if(hashes.begin(), hashes.end(), HashEntryWithPath(p.str()));
 841         if (it != hashes.end()) {
 842             hashes.erase(it);
 843         }
 844
 845         if (newHash.empty()) {
 846             return; // we're done
 847         }
 848
 849         // use a cloned SGPath and reset its caching to force one stat() call
 850         SGPath p2(p);
 851         p2.set_cached(false);
 852         p2.set_cached(true);
 853
 854         HashCacheEntry entry;
 855         entry.filePath = p.str();
 856         entry.hashHex = newHash;
 857         entry.modTime = p2.modTime();
 858         entry.lengthBytes = p2.sizeInBytes();
 859         hashes.push_back(entry);
 860
 861         writeHashCache();
 862     }
 863
 864     void HTTPRepoPrivate::writeHashCache()
 865     {
 866         SGPath cachePath = basePath;
 867         cachePath.append(".hashes");
 868
 869         std::ofstream stream(cachePath.c_str(),std::ios::out | std::ios::trunc);
 870         HashCache::const_iterator it;
 871         for (it = hashes.begin(); it != hashes.end(); ++it) {
 872             stream << it->filePath << ":" << it->modTime << ":"
 873             << it->lengthBytes << ":" << it->hashHex << "\n";
 874         }
 875         stream.close();
 876     }
 877
 878     void HTTPRepoPrivate::parseHashCache()
 879     {
 880         hashes.clear();
 881         SGPath cachePath = basePath;
 882         cachePath.append(".hashes");
 883         if (!cachePath.exists()) {
 884             return;
 885         }
 886
 887         std::ifstream stream(cachePath.c_str(), std::ios::in);
 888         char buf[2048];
 889         char* lastToken;
 890
 891         while (!stream.eof()) {
 892             stream.getline(buf, 2048);
 893             lastToken = 0;
 894             char* nameData = ::strtok_r(buf, ":", &lastToken);
 895             char* timeData = ::strtok_r(NULL, ":", &lastToken);
 896             char* sizeData = ::strtok_r(NULL, ":", &lastToken);
 897             char* hashData = ::strtok_r(NULL, ":", &lastToken);
 898             if (!nameData || !timeData || !sizeData || !hashData) {
 899                 continue;
 900             }
 901
 902             HashCacheEntry entry;
 903             entry.filePath = nameData;
 904             entry.hashHex = hashData;
 905             entry.modTime = strtol(timeData, NULL, 10);
 906             entry.lengthBytes = strtol(sizeData, NULL, 10);
 907             hashes.push_back(entry);
 908         }
 909     }
 910
 911     class DirectoryWithPath
 912     {
 913     public:
 914         DirectoryWithPath(const std::string& p) : path(p) {}
 915         bool operator()(const HTTPDirectory* entry) const
 916         { return entry->relativePath().str() == path; }
 917     private:
 918         std::string path;
 919     };
 920
 921     HTTPDirectory* HTTPRepoPrivate::getOrCreateDirectory(const std::string& path)
 922     {
 923         DirectoryWithPath p(path);
 924         DirectoryVector::iterator it = std::find_if(directories.begin(), directories.end(), p);
 925         if (it != directories.end()) {
 926             return *it;
 927         }
 928
 929         HTTPDirectory* d = new HTTPDirectory(this, path);
 930         directories.push_back(d);
 931         return d;
 932     }
 933
 934     bool HTTPRepoPrivate::deleteDirectory(const std::string& path)
 935     {
 936         DirectoryWithPath p(path);
 937         DirectoryVector::iterator it = std::find_if(directories.begin(), directories.end(), p);
 938         if (it != directories.end()) {
 939             HTTPDirectory* d = *it;
 940             directories.erase(it);
 941             Dir dir(d->absolutePath());
 942             bool result = dir.remove(true);
 943             delete d;
 944
 945             // update the hash cache too
 946             updatedFileContents(path, std::string());
 947
 948             return result;
 949         }
 950
 951         return false;
 952     }
 953
 954     void HTTPRepoPrivate::finishedRequest(const RepoRequestPtr& req)
 955     {
 956         RequestVector::iterator it = std::find(requests.begin(), requests.end(), req);
 957         if (it == requests.end()) {
 958             throw sg_exception("lost request somehow", req->url());
 959         }
 960         requests.erase(it);
 961         if (requests.empty()) {
 962             isUpdating = false;
 963         }
 964     }
 965
 966     void HTTPRepoPrivate::failedToGetRootIndex(AbstractRepository::ResultCode st)
 967     {
 968         SG_LOG(SG_TERRASYNC, SG_WARN, "Failed to get root of repo:" << baseUrl);
 969         status = st;
 970     }
 971
 972     void HTTPRepoPrivate::failedToUpdateChild(const SGPath& relativePath,
 973                                               AbstractRepository::ResultCode fileStatus)
 974     {
 975         Failure f;
 976         f.path = relativePath;
 977         f.error = fileStatus;
 978         failures.push_back(f);
 979
 980         SG_LOG(SG_TERRASYNC, SG_WARN, "failed to update entry:" << relativePath << " code:" << fileStatus);
 981     }
 982
 983
 984
 985 } // of namespace simgear