simgear/io/HTTPRepository.cxx

   1 // HTTPRepository.cxx -- plain HTTP TerraSync remote client
   2 //
   3 // Copyright (C) 20126  James Turner <zakalawe@mac.com>
   4 //
   5 // This program is free software; you can redistribute it and/or
   6 // modify it under the terms of the GNU General Public License as
   7 // published by the Free Software Foundation; either version 2 of the
   8 // License, or (at your option) any later version.
   9 //
  10 // This program is distributed in the hope that it will be useful, but
  11 // WITHOUT ANY WARRANTY; without even the implied warranty of
  12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 // General Public License for more details.
  14 //
  15 // You should have received a copy of the GNU General Public License
  16 // along with this program; if not, write to the Free Software
  17 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  18
  19 #include "HTTPRepository.hxx"
  20
  21 #include <simgear_config.h>
  22
  23 #include <iostream>
  24 #include <cstring>
  25 #include <cassert>
  26 #include <algorithm>
  27 #include <sstream>
  28 #include <map>
  29 #include <set>
  30 #include <fstream>
  31 #include <limits>
  32 #include <cstdlib>
  33
  34 #include <fcntl.h>
  35
  36 #include "simgear/debug/logstream.hxx"
  37 #include "simgear/misc/strutils.hxx"
  38 #include <simgear/misc/sg_dir.hxx>
  39 #include <simgear/io/HTTPClient.hxx>
  40 #include <simgear/io/sg_file.hxx>
  41 #include <simgear/misc/sgstream.hxx>
  42 #include <simgear/structure/exception.hxx>
  43 #include <simgear/timing/timestamp.hxx>
  44
  45 #include <simgear/misc/sg_hash.hxx>
  46
  47 #if defined(SG_WINDOWS)
  48
  49 /*
  50  * public domain strtok_r() by Charlie Gordon
  51  *
  52  *   from comp.lang.c  9/14/2007
  53  *
  54  *      http://groups.google.com/group/comp.lang.c/msg/2ab1ecbb86646684
  55  *
  56  *     (Declaration that it's public domain):
  57  *      http://groups.google.com/group/comp.lang.c/msg/7c7b39328fefab9c
  58  */
  59
  60 char* strtok_r(
  61     char *str,
  62     const char *delim,
  63     char **nextp)
  64 {
  65     char *ret;
  66
  67     if (str == NULL)
  68     {
  69         str = *nextp;
  70     }
  71
  72     str += strspn(str, delim);
  73
  74     if (*str == '\0')
  75     {
  76         return NULL;
  77     }
  78
  79     ret = str;
  80
  81     str += strcspn(str, delim);
  82
  83     if (*str)
  84     {
  85         *str++ = '\0';
  86     }
  87
  88     *nextp = str;
  89
  90     return ret;
  91 }
  92 #endif
  93
  94 namespace simgear
  95 {
  96
  97     class HTTPDirectory;
  98
  99     class HTTPRepoGetRequest : public HTTP::Request
 100     {
 101     public:
 102         HTTPRepoGetRequest(HTTPDirectory* d, const std::string& u) :
 103             HTTP::Request(u),
 104             _directory(d)
 105         {
 106         }
 107
 108         virtual void cancel();
 109
 110         size_t contentSize() const
 111         {
 112             return _contentSize;
 113         }
 114
 115         void setContentSize(size_t sz)
 116         {
 117             _contentSize = sz;
 118         }
 119     protected:
 120         HTTPDirectory* _directory;
 121         size_t _contentSize;
 122     };
 123
 124     typedef SGSharedPtr<HTTPRepoGetRequest> RepoRequestPtr;
 125
 126 class HTTPRepoPrivate
 127 {
 128 public:
 129     struct HashCacheEntry
 130     {
 131         std::string filePath;
 132         time_t modTime;
 133         size_t lengthBytes;
 134         std::string hashHex;
 135
 136     };
 137
 138     typedef std::vector<HashCacheEntry> HashCache;
 139     HashCache hashes;
 140     bool hashCacheDirty;
 141
 142     struct Failure
 143     {
 144         SGPath path;
 145         AbstractRepository::ResultCode error;
 146     };
 147
 148     typedef std::vector<Failure> FailureList;
 149     FailureList failures;
 150
 151     HTTPRepoPrivate(HTTPRepository* parent) :
 152         hashCacheDirty(false),
 153         p(parent),
 154         isUpdating(false),
 155         status(AbstractRepository::REPO_NO_ERROR),
 156         totalDownloaded(0)
 157     { ; }
 158
 159     ~HTTPRepoPrivate();
 160
 161     HTTPRepository* p; // link back to outer
 162     HTTP::Client* http;
 163     std::string baseUrl;
 164     SGPath basePath;
 165     bool isUpdating;
 166     AbstractRepository::ResultCode status;
 167     HTTPDirectory* rootDir;
 168     size_t totalDownloaded;
 169
 170     HTTP::Request_ptr updateFile(HTTPDirectory* dir, const std::string& name,
 171                                  size_t sz);
 172     HTTP::Request_ptr updateDir(HTTPDirectory* dir, const std::string& hash,
 173                                 size_t sz);
 174
 175     std::string hashForPath(const SGPath& p);
 176     void updatedFileContents(const SGPath& p, const std::string& newHash);
 177     void parseHashCache();
 178     std::string computeHashForPath(const SGPath& p);
 179     void writeHashCache();
 180
 181     void failedToGetRootIndex(AbstractRepository::ResultCode st);
 182     void failedToUpdateChild(const SGPath& relativePath,
 183                              AbstractRepository::ResultCode fileStatus);
 184
 185     typedef std::vector<RepoRequestPtr> RequestVector;
 186     RequestVector queuedRequests,
 187         activeRequests;
 188
 189     void makeRequest(RepoRequestPtr req);
 190     void finishedRequest(const RepoRequestPtr& req);
 191
 192     HTTPDirectory* getOrCreateDirectory(const std::string& path);
 193     bool deleteDirectory(const std::string& path);
 194
 195     typedef std::vector<HTTPDirectory*> DirectoryVector;
 196     DirectoryVector directories;
 197
 198 };
 199
 200 class HTTPDirectory
 201 {
 202     struct ChildInfo
 203     {
 204         enum Type
 205         {
 206             FileType,
 207             DirectoryType
 208         };
 209
 210         ChildInfo(Type ty, const char* nameData, const char* hashData) :
 211             type(ty),
 212             name(nameData),
 213             hash(hashData ? hashData : ""),
 214             sizeInBytes(0)
 215         {
 216         }
 217
 218         ChildInfo(const ChildInfo& other) :
 219             type(other.type),
 220             name(other.name),
 221             hash(other.hash),
 222             sizeInBytes(other.sizeInBytes)
 223         { }
 224
 225         void setSize(const char* sizeData)
 226         {
 227             sizeInBytes = ::strtol(sizeData, NULL, 10);
 228         }
 229
 230         bool operator<(const ChildInfo& other) const
 231         {
 232             return name < other.name;
 233         }
 234
 235         Type type;
 236         std::string name, hash;
 237         size_t sizeInBytes;
 238     };
 239
 240     typedef std::vector<ChildInfo> ChildInfoList;
 241     ChildInfoList children;
 242
 243 public:
 244     HTTPDirectory(HTTPRepoPrivate* repo, const std::string& path) :
 245         _repository(repo),
 246         _relativePath(path)
 247   {
 248       assert(repo);
 249
 250       SGPath p(absolutePath());
 251       if (p.exists()) {
 252           try {
 253               // already exists on disk
 254               parseDirIndex(children);
 255               std::sort(children.begin(), children.end());
 256           } catch (sg_exception& e) {
 257               // parsing cache failed
 258               children.clear();
 259           }
 260       }
 261   }
 262
 263     HTTPRepoPrivate* repository() const
 264     {
 265         return _repository;
 266     }
 267
 268     std::string url() const
 269     {
 270         if (_relativePath.str().empty()) {
 271             return _repository->baseUrl;
 272         }
 273
 274         return _repository->baseUrl + "/" + _relativePath.str();
 275     }
 276
 277     void dirIndexUpdated(const std::string& hash)
 278     {
 279         SGPath fpath(_relativePath);
 280         fpath.append(".dirindex");
 281         _repository->updatedFileContents(fpath, hash);
 282
 283         children.clear();
 284         parseDirIndex(children);
 285         std::sort(children.begin(), children.end());
 286     }
 287
 288     void failedToUpdate(AbstractRepository::ResultCode status)
 289     {
 290         if (_relativePath.isNull()) {
 291             // root dir failed
 292             _repository->failedToGetRootIndex(status);
 293         } else {
 294             _repository->failedToUpdateChild(_relativePath, status);
 295         }
 296     }
 297
 298     void updateChildrenBasedOnHash()
 299     {
 300         //SG_LOG(SG_TERRASYNC, SG_DEBUG, "updated children for:" << relativePath());
 301
 302         string_list indexNames = indexChildren(),
 303             toBeUpdated, orphans;
 304         simgear::Dir d(absolutePath());
 305         PathList fsChildren = d.children(0);
 306         PathList::const_iterator it = fsChildren.begin();
 307
 308         for (; it != fsChildren.end(); ++it) {
 309             ChildInfo info(it->isDir() ? ChildInfo::DirectoryType : ChildInfo::FileType,
 310                            it->file().c_str(), NULL);
 311             std::string hash = hashForChild(info);
 312
 313             ChildInfoList::iterator c = findIndexChild(it->file());
 314             if (c == children.end()) {
 315                 orphans.push_back(it->file());
 316             } else if (c->hash != hash) {
 317                 // file exists, but hash mismatch, schedule update
 318                 if (!hash.empty()) {
 319                     //SG_LOG(SG_TERRASYNC, SG_INFO, "file exists but hash is wrong for:" << c->name);
 320                     //SG_LOG(SG_TERRASYNC, SG_INFO, "on disk:" << hash << " vs in info:" << c->hash);
 321                 }
 322
 323                 toBeUpdated.push_back(c->name);
 324             } else {
 325                 // file exists and hash is valid. If it's a directory,
 326                 // perform a recursive check.
 327                 if (c->type == ChildInfo::DirectoryType) {
 328                     SGPath p(relativePath());
 329                     p.append(c->name);
 330                     HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str());
 331                     childDir->updateChildrenBasedOnHash();
 332                 }
 333             }
 334
 335             // remove existing file system children from the index list,
 336             // so we can detect new children
 337             string_list::iterator it = std::find(indexNames.begin(), indexNames.end(), c->name);
 338             if (it != indexNames.end()) {
 339                 indexNames.erase(it);
 340             }
 341         } // of real children iteration
 342
 343         // all remaining names in indexChilden are new children
 344         toBeUpdated.insert(toBeUpdated.end(), indexNames.begin(), indexNames.end());
 345
 346         removeOrphans(orphans);
 347         scheduleUpdates(toBeUpdated);
 348     }
 349
 350     void removeOrphans(const string_list& orphans)
 351     {
 352         string_list::const_iterator it;
 353         for (it = orphans.begin(); it != orphans.end(); ++it) {
 354             removeChild(*it);
 355         }
 356     }
 357
 358     string_list indexChildren() const
 359     {
 360         string_list r;
 361         r.reserve(children.size());
 362         ChildInfoList::const_iterator it;
 363         for (it=children.begin(); it != children.end(); ++it) {
 364             r.push_back(it->name);
 365         }
 366         return r;
 367     }
 368
 369     void scheduleUpdates(const string_list& names)
 370     {
 371         string_list::const_iterator it;
 372         for (it = names.begin(); it != names.end(); ++it) {
 373             ChildInfoList::iterator cit = findIndexChild(*it);
 374             if (cit == children.end()) {
 375                 SG_LOG(SG_TERRASYNC, SG_WARN, "scheduleUpdate, unknown child:" << *it);
 376                 continue;
 377             }
 378
 379             if (cit->type == ChildInfo::FileType) {
 380                 _repository->updateFile(this, *it, cit->sizeInBytes);
 381             } else {
 382                 SGPath p(relativePath());
 383                 p.append(*it);
 384                 HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str());
 385                 _repository->updateDir(childDir, cit->hash, cit->sizeInBytes);
 386             }
 387         }
 388     }
 389
 390     SGPath absolutePath() const
 391     {
 392         SGPath r(_repository->basePath);
 393         r.append(_relativePath.str());
 394         return r;
 395     }
 396
 397     SGPath relativePath() const
 398     {
 399         return _relativePath;
 400     }
 401
 402     void didUpdateFile(const std::string& file, const std::string& hash, size_t sz)
 403     {
 404         // check hash matches what we expected
 405         ChildInfoList::iterator it = findIndexChild(file);
 406         if (it == children.end()) {
 407             SG_LOG(SG_TERRASYNC, SG_WARN, "updated file but not found in dir:" << _relativePath << " " << file);
 408         } else {
 409             SGPath fpath(_relativePath);
 410             fpath.append(file);
 411
 412             if (it->hash != hash) {
 413                 _repository->failedToUpdateChild(_relativePath, AbstractRepository::REPO_ERROR_CHECKSUM);
 414             } else {
 415                 _repository->updatedFileContents(fpath, hash);
 416                 _repository->totalDownloaded += sz;
 417                 //SG_LOG(SG_TERRASYNC, SG_INFO, "did update:" << fpath);
 418             } // of hash matches
 419         } // of found in child list
 420     }
 421
 422     void didFailToUpdateFile(const std::string& file,
 423                              AbstractRepository::ResultCode status)
 424     {
 425         SGPath fpath(_relativePath);
 426         fpath.append(file);
 427         _repository->failedToUpdateChild(fpath, status);
 428     }
 429 private:
 430
 431     struct ChildWithName
 432     {
 433         ChildWithName(const std::string& n) : name(n) {}
 434         std::string name;
 435
 436         bool operator()(const ChildInfo& info) const
 437         { return info.name == name; }
 438     };
 439
 440     ChildInfoList::iterator findIndexChild(const std::string& name)
 441     {
 442         return std::find_if(children.begin(), children.end(), ChildWithName(name));
 443     }
 444
 445     bool parseDirIndex(ChildInfoList& children)
 446     {
 447         SGPath p(absolutePath());
 448         p.append(".dirindex");
 449         if (!p.exists()) {
 450             return false;
 451         }
 452
 453         std::ifstream indexStream( p.c_str(), std::ios::in );
 454
 455         if ( !indexStream.is_open() ) {
 456             throw sg_io_exception("cannot open dirIndex file", p);
 457         }
 458
 459         char lineBuffer[512];
 460         char* lastToken;
 461
 462         while (!indexStream.eof() ) {
 463             indexStream.getline(lineBuffer, 512);
 464             lastToken = 0;
 465             char* typeData = ::strtok_r(lineBuffer, ":", &lastToken);
 466             if (!typeData) {
 467                 continue; // skip blank line
 468             }
 469
 470             if (!typeData) {
 471                 // malformed entry
 472                 throw sg_io_exception("Malformed dir index file", p);
 473             }
 474
 475             if (!strcmp(typeData, "version")) {
 476                 continue;
 477             } else if (!strcmp(typeData, "path")) {
 478                 continue;
 479             }
 480
 481             char* nameData = ::strtok_r(NULL, ":", &lastToken);
 482             char* hashData = ::strtok_r(NULL, ":", &lastToken);
 483             char* sizeData = ::strtok_r(NULL, ":", &lastToken);
 484
 485             if (typeData[0] == 'f') {
 486                 children.push_back(ChildInfo(ChildInfo::FileType, nameData, hashData));
 487             } else if (typeData[0] == 'd') {
 488                 children.push_back(ChildInfo(ChildInfo::DirectoryType, nameData, hashData));
 489             } else {
 490                 throw sg_io_exception("Malformed line code in dir index file", p);
 491             }
 492
 493             if (sizeData) {
 494                 children.back().setSize(sizeData);
 495             }
 496         }
 497
 498         return true;
 499     }
 500
 501     void removeChild(const std::string& name)
 502     {
 503         SGPath p(absolutePath());
 504         p.append(name);
 505         bool ok;
 506
 507         SGPath fpath(_relativePath);
 508         fpath.append(name);
 509
 510         if (p.isDir()) {
 511             ok = _repository->deleteDirectory(fpath.str());
 512         } else {
 513             // remove the hash cache entry
 514             _repository->updatedFileContents(fpath, std::string());
 515             ok = p.remove();
 516         }
 517
 518         if (!ok) {
 519             SG_LOG(SG_TERRASYNC, SG_WARN, "removal failed for:" << p);
 520             throw sg_io_exception("Failed to remove existing file/dir:", p);
 521         }
 522     }
 523
 524     std::string hashForChild(const ChildInfo& child) const
 525     {
 526         SGPath p(absolutePath());
 527         p.append(child.name);
 528         if (child.type == ChildInfo::DirectoryType) {
 529             p.append(".dirindex");
 530         }
 531         return _repository->hashForPath(p);
 532     }
 533
 534   HTTPRepoPrivate* _repository;
 535   SGPath _relativePath; // in URL and file-system space
 536
 537
 538 };
 539
 540 HTTPRepository::HTTPRepository(const SGPath& base, HTTP::Client *cl) :
 541     _d(new HTTPRepoPrivate(this))
 542 {
 543     _d->http = cl;
 544     _d->basePath = base;
 545     _d->rootDir = new HTTPDirectory(_d.get(), "");
 546     _d->parseHashCache();
 547 }
 548
 549 HTTPRepository::~HTTPRepository()
 550 {
 551 }
 552
 553 void HTTPRepository::setBaseUrl(const std::string &url)
 554 {
 555   _d->baseUrl = url;
 556 }
 557
 558 std::string HTTPRepository::baseUrl() const
 559 {
 560   return _d->baseUrl;
 561 }
 562
 563 HTTP::Client* HTTPRepository::http() const
 564 {
 565   return _d->http;
 566 }
 567
 568 SGPath HTTPRepository::fsBase() const
 569 {
 570   return SGPath();
 571 }
 572
 573 void HTTPRepository::update()
 574 {
 575     if (_d->isUpdating) {
 576         return;
 577     }
 578
 579     _d->status = REPO_NO_ERROR;
 580     _d->isUpdating = true;
 581     _d->failures.clear();
 582     _d->updateDir(_d->rootDir, std::string(), 0);
 583 }
 584
 585 bool HTTPRepository::isDoingSync() const
 586 {
 587     if (_d->status != REPO_NO_ERROR) {
 588         return false;
 589     }
 590
 591     return _d->isUpdating;
 592 }
 593
 594 size_t HTTPRepository::bytesToDownload() const
 595 {
 596     size_t result = 0;
 597
 598     HTTPRepoPrivate::RequestVector::const_iterator r;
 599     for (r = _d->queuedRequests.begin(); r != _d->queuedRequests.end(); ++r) {
 600         result += (*r)->contentSize();
 601     }
 602
 603     for (r = _d->activeRequests.begin(); r != _d->activeRequests.end(); ++r) {
 604         result += (*r)->contentSize() - (*r)->responseBytesReceived();
 605     }
 606
 607     return result;
 608 }
 609
 610 size_t HTTPRepository::bytesDownloaded() const
 611 {
 612     size_t result = _d->totalDownloaded;
 613
 614     HTTPRepoPrivate::RequestVector::const_iterator r;
 615     for (r = _d->activeRequests.begin(); r != _d->activeRequests.end(); ++r) {
 616         result += (*r)->responseBytesReceived();
 617     }
 618
 619     return result;
 620 }
 621
 622 AbstractRepository::ResultCode
 623 HTTPRepository::failure() const
 624 {
 625     if ((_d->status == REPO_NO_ERROR) && !_d->failures.empty()) {
 626         return REPO_PARTIAL_UPDATE;
 627     }
 628
 629     return _d->status;
 630 }
 631
 632     void HTTPRepoGetRequest::cancel()
 633     {
 634         _directory->repository()->http->cancelRequest(this, "Reposiotry cancelled");
 635         _directory = 0;
 636     }
 637
 638     class FileGetRequest : public HTTPRepoGetRequest
 639     {
 640     public:
 641         FileGetRequest(HTTPDirectory* d, const std::string& file) :
 642             HTTPRepoGetRequest(d, makeUrl(d, file)),
 643             fileName(file)
 644         {
 645             pathInRepo = _directory->absolutePath();
 646             pathInRepo.append(fileName);
 647             //SG_LOG(SG_TERRASYNC, SG_INFO, "will GET file " << url());
 648         }
 649
 650     protected:
 651         virtual void gotBodyData(const char* s, int n)
 652         {
 653             if (!file.get()) {
 654                 file.reset(new SGFile(pathInRepo.str()));
 655                 if (!file->open(SG_IO_OUT)) {
 656                   SG_LOG(SG_TERRASYNC, SG_WARN, "unable to create file " << pathInRepo);
 657                   _directory->repository()->http->cancelRequest(this, "Unable to create output file");
 658                 }
 659
 660                 sha1_init(&hashContext);
 661             }
 662
 663             sha1_write(&hashContext, s, n);
 664             file->write(s, n);
 665         }
 666
 667         virtual void onDone()
 668         {
 669             file->close();
 670             if (responseCode() == 200) {
 671                 std::string hash = strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
 672                 _directory->didUpdateFile(fileName, hash, contentSize());
 673                 //SG_LOG(SG_TERRASYNC, SG_INFO, "got file " << fileName << " in " << _directory->absolutePath());
 674             } else if (responseCode() == 404) {
 675                 _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_FILE_NOT_FOUND);
 676             } else {
 677                 _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_HTTP);
 678             }
 679
 680             _directory->repository()->finishedRequest(this);
 681         }
 682
 683         virtual void onFail()
 684         {
 685             file.reset();
 686             if (pathInRepo.exists()) {
 687                 pathInRepo.remove();
 688             }
 689
 690             if (_directory) {
 691                 _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_SOCKET);
 692                 _directory->repository()->finishedRequest(this);
 693             }
 694         }
 695     private:
 696         static std::string makeUrl(HTTPDirectory* d, const std::string& file)
 697         {
 698             return d->url() + "/" + file;
 699         }
 700
 701         std::string fileName; // if empty, we're getting the directory itself
 702         SGPath pathInRepo;
 703         simgear::sha1nfo hashContext;
 704         std::auto_ptr<SGFile> file;
 705     };
 706
 707     class DirGetRequest : public HTTPRepoGetRequest
 708     {
 709     public:
 710         DirGetRequest(HTTPDirectory* d, const std::string& targetHash) :
 711             HTTPRepoGetRequest(d, makeUrl(d)),
 712             _isRootDir(false),
 713             _targetHash(targetHash)
 714         {
 715             sha1_init(&hashContext);
 716            //SG_LOG(SG_TERRASYNC, SG_INFO, "will GET dir " << url());
 717         }
 718
 719         void setIsRootDir()
 720         {
 721             _isRootDir = true;
 722         }
 723
 724         bool isRootDir() const
 725         {
 726             return _isRootDir;
 727         }
 728
 729     protected:
 730         virtual void gotBodyData(const char* s, int n)
 731         {
 732             body += std::string(s, n);
 733             sha1_write(&hashContext, s, n);
 734         }
 735
 736         virtual void onDone()
 737         {
 738             if (responseCode() == 200) {
 739                 std::string hash = strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
 740                 if (!_targetHash.empty() && (hash != _targetHash)) {
 741                     _directory->failedToUpdate(AbstractRepository::REPO_ERROR_CHECKSUM);
 742                     _directory->repository()->finishedRequest(this);
 743                     return;
 744                 }
 745
 746                 std::string curHash = _directory->repository()->hashForPath(path());
 747                 if (hash != curHash) {
 748                     simgear::Dir d(_directory->absolutePath());
 749                     if (!d.exists()) {
 750                         if (!d.create(0700)) {
 751                             throw sg_io_exception("Unable to create directory", d.path());
 752                         }
 753                     }
 754
 755                     // dir index data has changed, so write to disk and update
 756                     // the hash accordingly
 757                     std::ofstream of(pathInRepo().c_str(), std::ios::trunc | std::ios::out);
 758                     if (!of.is_open()) {
 759                         throw sg_io_exception("Failed to open directory index file for writing", pathInRepo().c_str());
 760                     }
 761
 762                     of.write(body.data(), body.size());
 763                     of.close();
 764                     _directory->dirIndexUpdated(hash);
 765
 766                     //SG_LOG(SG_TERRASYNC, SG_INFO, "updated dir index " << _directory->absolutePath());
 767                 }
 768
 769                 _directory->repository()->totalDownloaded += contentSize();
 770
 771                 try {
 772                     // either way we've confirmed the index is valid so update
 773                     // children now
 774                     SGTimeStamp st;
 775                     st.stamp();
 776                     _directory->updateChildrenBasedOnHash();
 777                     SG_LOG(SG_TERRASYNC, SG_INFO, "after update of:" << _directory->absolutePath() << " child update took:" << st.elapsedMSec());
 778                 } catch (sg_exception& e) {
 779                     _directory->failedToUpdate(AbstractRepository::REPO_ERROR_IO);
 780                 }
 781             } else if (responseCode() == 404) {
 782                 _directory->failedToUpdate(AbstractRepository::REPO_ERROR_FILE_NOT_FOUND);
 783             } else {
 784                 _directory->failedToUpdate(AbstractRepository::REPO_ERROR_HTTP);
 785             }
 786
 787             _directory->repository()->finishedRequest(this);
 788         }
 789
 790         virtual void onFail()
 791         {
 792             if (_directory) {
 793                 _directory->failedToUpdate(AbstractRepository::REPO_ERROR_SOCKET);
 794                 _directory->repository()->finishedRequest(this);
 795             }
 796         }
 797     private:
 798         static std::string makeUrl(HTTPDirectory* d)
 799         {
 800             return d->url() + "/.dirindex";
 801         }
 802
 803         SGPath pathInRepo() const
 804         {
 805             SGPath p(_directory->absolutePath());
 806             p.append(".dirindex");
 807             return p;
 808         }
 809
 810         simgear::sha1nfo hashContext;
 811         std::string body;
 812         bool _isRootDir; ///< is this the repository root?
 813         std::string _targetHash;
 814     };
 815
 816     HTTPRepoPrivate::~HTTPRepoPrivate()
 817     {
 818         DirectoryVector::iterator it;
 819         for (it=directories.begin(); it != directories.end(); ++it) {
 820             delete *it;
 821         }
 822
 823         RequestVector::iterator r;
 824         for (r=activeRequests.begin(); r != activeRequests.end(); ++r) {
 825             (*r)->cancel();
 826         }
 827     }
 828
 829     HTTP::Request_ptr HTTPRepoPrivate::updateFile(HTTPDirectory* dir, const std::string& name, size_t sz)
 830     {
 831         RepoRequestPtr r(new FileGetRequest(dir, name));
 832         r->setContentSize(sz);
 833         makeRequest(r);
 834         return r;
 835     }
 836
 837     HTTP::Request_ptr HTTPRepoPrivate::updateDir(HTTPDirectory* dir, const std::string& hash, size_t sz)
 838     {
 839         RepoRequestPtr r(new DirGetRequest(dir, hash));
 840         r->setContentSize(sz);
 841         makeRequest(r);
 842         return r;
 843     }
 844
 845
 846     class HashEntryWithPath
 847     {
 848     public:
 849         HashEntryWithPath(const std::string& p) : path(p) {}
 850         bool operator()(const HTTPRepoPrivate::HashCacheEntry& entry) const
 851         { return entry.filePath == path; }
 852     private:
 853         std::string path;
 854     };
 855
 856     std::string HTTPRepoPrivate::hashForPath(const SGPath& p)
 857     {
 858         HashCache::iterator it = std::find_if(hashes.begin(), hashes.end(), HashEntryWithPath(p.str()));
 859         if (it != hashes.end()) {
 860             // ensure data on disk hasn't changed.
 861             // we could also use the file type here if we were paranoid
 862             if ((p.sizeInBytes() == it->lengthBytes) && (p.modTime() == it->modTime)) {
 863                 return it->hashHex;
 864             }
 865
 866             // entry in the cache, but it's stale so remove and fall through
 867             hashes.erase(it);
 868         }
 869
 870         std::string hash = computeHashForPath(p);
 871         updatedFileContents(p, hash);
 872         return hash;
 873     }
 874
 875     std::string HTTPRepoPrivate::computeHashForPath(const SGPath& p)
 876     {
 877         if (!p.exists())
 878             return std::string();
 879         sha1nfo info;
 880         sha1_init(&info);
 881         char* buf = static_cast<char*>(malloc(1024 * 1024));
 882         size_t readLen;
 883         SGFile f(p.str());
 884         if (!f.open(SG_IO_IN)) {
 885             throw sg_io_exception("Couldn't open file for compute hash", p);
 886         }
 887         while ((readLen = f.read(buf, 1024 * 1024)) > 0) {
 888             sha1_write(&info, buf, readLen);
 889         }
 890
 891         f.close();
 892         free(buf);
 893         std::string hashBytes((char*) sha1_result(&info), HASH_LENGTH);
 894         return strutils::encodeHex(hashBytes);
 895     }
 896
 897     void HTTPRepoPrivate::updatedFileContents(const SGPath& p, const std::string& newHash)
 898     {
 899         // remove the existing entry
 900         HashCache::iterator it = std::find_if(hashes.begin(), hashes.end(), HashEntryWithPath(p.str()));
 901         if (it != hashes.end()) {
 902             hashes.erase(it);
 903             hashCacheDirty = true;
 904         }
 905
 906         if (newHash.empty()) {
 907             return; // we're done
 908         }
 909
 910         // use a cloned SGPath and reset its caching to force one stat() call
 911         SGPath p2(p);
 912         p2.set_cached(false);
 913         p2.set_cached(true);
 914
 915         HashCacheEntry entry;
 916         entry.filePath = p.str();
 917         entry.hashHex = newHash;
 918         entry.modTime = p2.modTime();
 919         entry.lengthBytes = p2.sizeInBytes();
 920         hashes.push_back(entry);
 921
 922         hashCacheDirty = true;
 923     }
 924
 925     void HTTPRepoPrivate::writeHashCache()
 926     {
 927         if (!hashCacheDirty) {
 928             return;
 929         }
 930
 931         SGPath cachePath = basePath;
 932         cachePath.append(".hashes");
 933
 934         std::ofstream stream(cachePath.c_str(),std::ios::out | std::ios::trunc);
 935         HashCache::const_iterator it;
 936         for (it = hashes.begin(); it != hashes.end(); ++it) {
 937             stream << it->filePath << ":" << it->modTime << ":"
 938             << it->lengthBytes << ":" << it->hashHex << "\n";
 939         }
 940         stream.close();
 941         hashCacheDirty = false;
 942     }
 943
 944     void HTTPRepoPrivate::parseHashCache()
 945     {
 946         hashes.clear();
 947         SGPath cachePath = basePath;
 948         cachePath.append(".hashes");
 949         if (!cachePath.exists()) {
 950             return;
 951         }
 952
 953         std::ifstream stream(cachePath.c_str(), std::ios::in);
 954         char buf[2048];
 955         char* lastToken;
 956
 957         while (!stream.eof()) {
 958             stream.getline(buf, 2048);
 959             lastToken = 0;
 960             char* nameData = ::strtok_r(buf, ":", &lastToken);
 961             char* timeData = ::strtok_r(NULL, ":", &lastToken);
 962             char* sizeData = ::strtok_r(NULL, ":", &lastToken);
 963             char* hashData = ::strtok_r(NULL, ":", &lastToken);
 964             if (!nameData || !timeData || !sizeData || !hashData) {
 965                 continue;
 966             }
 967
 968             HashCacheEntry entry;
 969             entry.filePath = nameData;
 970             entry.hashHex = hashData;
 971             entry.modTime = strtol(timeData, NULL, 10);
 972             entry.lengthBytes = strtol(sizeData, NULL, 10);
 973             hashes.push_back(entry);
 974         }
 975     }
 976
 977     class DirectoryWithPath
 978     {
 979     public:
 980         DirectoryWithPath(const std::string& p) : path(p) {}
 981         bool operator()(const HTTPDirectory* entry) const
 982         { return entry->relativePath().str() == path; }
 983     private:
 984         std::string path;
 985     };
 986
 987     HTTPDirectory* HTTPRepoPrivate::getOrCreateDirectory(const std::string& path)
 988     {
 989         DirectoryWithPath p(path);
 990         DirectoryVector::iterator it = std::find_if(directories.begin(), directories.end(), p);
 991         if (it != directories.end()) {
 992             return *it;
 993         }
 994
 995         HTTPDirectory* d = new HTTPDirectory(this, path);
 996         directories.push_back(d);
 997         return d;
 998     }
 999
1000     bool HTTPRepoPrivate::deleteDirectory(const std::string& path)
1001     {
1002         DirectoryWithPath p(path);
1003         DirectoryVector::iterator it = std::find_if(directories.begin(), directories.end(), p);
1004         if (it != directories.end()) {
1005             HTTPDirectory* d = *it;
1006             directories.erase(it);
1007             Dir dir(d->absolutePath());
1008             bool result = dir.remove(true);
1009             delete d;
1010
1011             // update the hash cache too
1012             updatedFileContents(path, std::string());
1013
1014             return result;
1015         }
1016
1017         return false;
1018     }
1019
1020     void HTTPRepoPrivate::makeRequest(RepoRequestPtr req)
1021     {
1022         if (activeRequests.size() > 4) {
1023             queuedRequests.push_back(req);
1024         } else {
1025             activeRequests.push_back(req);
1026             http->makeRequest(req);
1027         }
1028     }
1029
1030     void HTTPRepoPrivate::finishedRequest(const RepoRequestPtr& req)
1031     {
1032         RequestVector::iterator it = std::find(activeRequests.begin(), activeRequests.end(), req);
1033         if (it == activeRequests.end()) {
1034             throw sg_exception("lost request somehow", req->url());
1035         }
1036         activeRequests.erase(it);
1037
1038         if (!queuedRequests.empty()) {
1039             RepoRequestPtr rr = queuedRequests.front();
1040             queuedRequests.erase(queuedRequests.begin());
1041             activeRequests.push_back(rr);
1042             http->makeRequest(rr);
1043         }
1044
1045         writeHashCache();
1046
1047         if (activeRequests.empty() && queuedRequests.empty()) {
1048             isUpdating = false;
1049         }
1050     }
1051
1052     void HTTPRepoPrivate::failedToGetRootIndex(AbstractRepository::ResultCode st)
1053     {
1054         SG_LOG(SG_TERRASYNC, SG_WARN, "Failed to get root of repo:" << baseUrl);
1055         status = st;
1056     }
1057
1058     void HTTPRepoPrivate::failedToUpdateChild(const SGPath& relativePath,
1059                                               AbstractRepository::ResultCode fileStatus)
1060     {
1061         Failure f;
1062         f.path = relativePath;
1063         f.error = fileStatus;
1064         failures.push_back(f);
1065
1066         SG_LOG(SG_TERRASYNC, SG_WARN, "failed to update entry:" << relativePath << " code:" << fileStatus);
1067     }
1068
1069
1070
1071 } // of namespace simgear