simgear/io/test_repository.cxx

   1 #include <cstdlib>
   2 #include <iostream>
   3 #include <map>
   4 #include <sstream>
   5 #include <errno.h>
   6 #include <fcntl.h>
   7
   8 #include <boost/algorithm/string/case_conv.hpp>
   9
  10 #include <simgear/simgear_config.h>
  11
  12 #include "test_HTTP.hxx"
  13 #include "HTTPRepository.hxx"
  14 #include "HTTPClient.hxx"
  15
  16 #include <simgear/misc/strutils.hxx>
  17 #include <simgear/misc/sg_hash.hxx>
  18 #include <simgear/timing/timestamp.hxx>
  19 #include <simgear/debug/logstream.hxx>
  20 #include <simgear/misc/sg_dir.hxx>
  21 #include <simgear/structure/exception.hxx>
  22 #include <simgear/io/sg_file.hxx>
  23
  24 using namespace simgear;
  25
  26 std::string dataForFile(const std::string& parentName, const std::string& name, int revision)
  27 {
  28     std::ostringstream os;
  29     // random content but which definitely depends on our tree location
  30     // and revision.
  31     for (int i=0; i<100; ++i) {
  32         os << i << parentName << "_" << name << "_" << revision;
  33     }
  34
  35     return os.str();
  36 }
  37
  38 std::string hashForData(const std::string& d)
  39 {
  40     simgear::sha1nfo info;
  41     sha1_init(&info);
  42     sha1_write(&info, d.data(), d.size());
  43     return strutils::encodeHex(sha1_result(&info), HASH_LENGTH);
  44 }
  45
  46 class TestRepoEntry
  47 {
  48 public:
  49     TestRepoEntry(TestRepoEntry* parent, const std::string& name, bool isDir);
  50     ~TestRepoEntry();
  51
  52     TestRepoEntry* parent;
  53     std::string name;
  54
  55     std::string indexLine() const;
  56
  57     std::string hash() const;
  58
  59     std::vector<TestRepoEntry*> children;
  60
  61     size_t sizeInBytes() const
  62     {
  63         return data().size();
  64     }
  65
  66     bool isDir;
  67     int revision; // for files
  68     int requestCount;
  69     bool getWillFail;
  70     bool returnCorruptData;
  71
  72     void clearRequestCounts();
  73
  74     void setGetWillFail(bool b)
  75     {
  76         getWillFail = b;
  77     }
  78
  79     void setReturnCorruptData(bool d)
  80     {
  81         returnCorruptData = d;
  82     }
  83
  84     std::string pathInRepo() const
  85     {
  86         return parent ? (parent->pathInRepo() + "/" + name) : name;
  87     }
  88
  89     std::string data() const;
  90
  91     void defineFile(const std::string& path, int rev = 1)
  92     {
  93         string_list pathParts = strutils::split(path, "/");
  94         if (pathParts.size() == 1) {
  95             children.push_back(new TestRepoEntry(this, pathParts.front(), false));
  96             children.back()->revision = rev;
  97         } else {
  98             // recurse
  99             TestRepoEntry* c = childEntry(pathParts.front());
 100             if (!c) {
 101                 // define a new directory child
 102                 c = new TestRepoEntry(this, pathParts.front(), true);
 103                 children.push_back(c);
 104             }
 105
 106             size_t frontPartLength = pathParts.front().size();
 107             c->defineFile(path.substr(frontPartLength + 1), rev);
 108         }
 109     }
 110
 111     TestRepoEntry* findEntry(const std::string& path)
 112     {
 113         if (path.empty()) {
 114             return this;
 115         }
 116
 117         string_list pathParts = strutils::split(path, "/");
 118         TestRepoEntry* entry = childEntry(pathParts.front());
 119         if (pathParts.size() == 1) {
 120             return entry; // might be NULL
 121         }
 122
 123         if (!entry) {
 124             std::cerr << "bad path: " << path << std::endl;
 125             return NULL;
 126         }
 127
 128         size_t part0Length = pathParts.front().size() + 1;
 129         return entry->findEntry(path.substr(part0Length));
 130     }
 131
 132     TestRepoEntry* childEntry(const std::string& name) const
 133     {
 134         assert(isDir);
 135         for (size_t i=0; i<children.size(); ++i) {
 136             if (children[i]->name == name) {
 137                 return children[i];
 138             }
 139         }
 140
 141         return NULL;
 142     }
 143
 144     void removeChild(const std::string& name)
 145     {
 146         std::vector<TestRepoEntry*>::iterator it;
 147         for (it = children.begin(); it != children.end(); ++it) {
 148             if ((*it)->name == name) {
 149                 delete *it;
 150                 children.erase(it);
 151                 return;
 152             }
 153         }
 154         std::cerr << "child not found:" << name << std::endl;
 155     }
 156 };
 157
 158 TestRepoEntry::TestRepoEntry(TestRepoEntry* pr, const std::string& nm, bool d) :
 159     parent(pr), name(nm), isDir(d)
 160 {
 161     revision = 2;
 162     requestCount = 0;
 163     getWillFail = false;
 164     returnCorruptData = false;
 165 }
 166
 167 TestRepoEntry::~TestRepoEntry()
 168 {
 169     for (size_t i=0; i<children.size(); ++i) {
 170         delete children[i];
 171     }
 172 }
 173
 174 std::string TestRepoEntry::data() const
 175 {
 176     if (isDir) {
 177         std::ostringstream os;
 178         os << "version:1\n";
 179         os << "path:" << pathInRepo() << "\n";
 180         for (size_t i=0; i<children.size(); ++i) {
 181             os << children[i]->indexLine() << "\n";
 182         }
 183         return os.str();
 184     } else {
 185         return dataForFile(parent->name, name, revision);
 186     }
 187 }
 188
 189 std::string TestRepoEntry::indexLine() const
 190 {
 191     std::ostringstream os;
 192     os << (isDir ? "d:" : "f:") << name << ":" << hash()
 193         << ":" << sizeInBytes();
 194     return os.str();
 195 }
 196
 197 std::string TestRepoEntry::hash() const
 198 {
 199     simgear::sha1nfo info;
 200     sha1_init(&info);
 201     std::string d(data());
 202     sha1_write(&info, d.data(), d.size());
 203     return strutils::encodeHex(sha1_result(&info), HASH_LENGTH);
 204 }
 205
 206 void TestRepoEntry::clearRequestCounts()
 207 {
 208     requestCount = 0;
 209     if (isDir) {
 210         for (size_t i=0; i<children.size(); ++i) {
 211             children[i]->clearRequestCounts();
 212         }
 213     }
 214 }
 215
 216 TestRepoEntry* global_repo = NULL;
 217
 218 class TestRepositoryChannel : public TestServerChannel
 219 {
 220 public:
 221
 222     virtual void processRequestHeaders()
 223     {
 224         state = STATE_IDLE;
 225         if (path.find("/repo/") == 0) {
 226 //            std::cerr << "get for:" << path << std::endl;
 227
 228             std::string repoPath = path.substr(6);
 229             bool lookingForDir = false;
 230             std::string::size_type suffix = repoPath.find(".dirindex");
 231             if (suffix != std::string::npos) {
 232                 lookingForDir = true;
 233                 if (suffix > 0) {
 234                     // trim the preceeding '/' as well, for non-root dirs
 235                     suffix--;
 236                 }
 237
 238                 repoPath = repoPath.substr(0, suffix);
 239             }
 240
 241             TestRepoEntry* entry = global_repo->findEntry(repoPath);
 242             if (!entry) {
 243                 sendErrorResponse(404, false, "unknown repo path:" + repoPath);
 244                 return;
 245             }
 246
 247             if (entry->isDir != lookingForDir) {
 248                 sendErrorResponse(404, false, "mismatched path type:" + repoPath);
 249                 return;
 250             }
 251
 252             if (entry->getWillFail) {
 253                 sendErrorResponse(404, false, "entry marked to fail explicitly:" + repoPath);
 254                 return;
 255             }
 256
 257             entry->requestCount++;
 258
 259             std::string content;
 260             if (entry->returnCorruptData) {
 261                 content = dataForFile("!$£$!" + entry->parent->name,
 262                                       "corrupt_" + entry->name,
 263                                       entry->revision);
 264             } else {
 265                 content = entry->data();
 266             }
 267
 268             std::stringstream d;
 269             d << "HTTP/1.1 " << 200 << " " << reasonForCode(200) << "\r\n";
 270             d << "Content-Length:" << content.size() << "\r\n";
 271             d << "\r\n"; // final CRLF to terminate the headers
 272             d << content;
 273             push(d.str().c_str());
 274         } else {
 275             sendErrorResponse(404, false, "");
 276         }
 277     }
 278 };
 279
 280 std::string test_computeHashForPath(const SGPath& p)
 281 {
 282     if (!p.exists())
 283         return std::string();
 284     sha1nfo info;
 285     sha1_init(&info);
 286     char* buf = static_cast<char*>(alloca(1024 * 1024));
 287     size_t readLen;
 288
 289     SGFile f(p.str());
 290     f.open(SG_IO_IN);
 291
 292     while ((readLen = f.read(buf, 1024 * 1024)) > 0) {
 293         sha1_write(&info, buf, readLen);
 294     }
 295
 296     std::string hashBytes((char*) sha1_result(&info), HASH_LENGTH);
 297     return strutils::encodeHex(hashBytes);
 298 }
 299
 300 void verifyFileState(const SGPath& fsRoot, const std::string& relPath)
 301 {
 302     TestRepoEntry* entry = global_repo->findEntry(relPath);
 303     if (!entry) {
 304         throw sg_error("Missing test repo entry", relPath);
 305     }
 306
 307     SGPath p(fsRoot);
 308     p.append(relPath);
 309     if (!p.exists()) {
 310         throw sg_error("Missing file system entry", relPath);
 311     }
 312
 313     std::string hashOnDisk = test_computeHashForPath(p);
 314     if (hashOnDisk != entry->hash()) {
 315         throw sg_error("Checksum mismatch", relPath);
 316     }
 317 }
 318
 319 void verifyRequestCount(const std::string& relPath, int count)
 320 {
 321     TestRepoEntry* entry = global_repo->findEntry(relPath);
 322     if (!entry) {
 323         throw sg_error("Missing test repo entry", relPath);
 324     }
 325
 326     if (entry->requestCount != count) {
 327         throw sg_exception("Bad request count", relPath);
 328     }
 329 }
 330
 331 void createFile(const SGPath& basePath, const std::string& relPath, int revision)
 332 {
 333     string_list comps = strutils::split(relPath, "/");
 334
 335     SGPath p(basePath);
 336     p.append(relPath);
 337
 338     simgear::Dir d(p.dir());
 339     d.create(0700);
 340
 341     std::string prName = comps.at(comps.size() - 2);
 342     {
 343         std::ofstream f(p.c_str(), std::ios::trunc | std::ios::out);
 344         f << dataForFile(prName, comps.back(), revision);
 345     }
 346 }
 347
 348 TestServer<TestRepositoryChannel> testServer;
 349
 350 void waitForUpdateComplete(HTTP::Client* cl, HTTPRepository* repo)
 351 {
 352     SGTimeStamp start(SGTimeStamp::now());
 353     while (start.elapsedMSec() <  10000) {
 354         cl->update();
 355         testServer.poll();
 356
 357         if (!repo->isDoingSync()) {
 358             return;
 359         }
 360         SGTimeStamp::sleepForMSec(15);
 361     }
 362
 363     std::cerr << "timed out" << std::endl;
 364 }
 365
 366 void testBasicClone(HTTP::Client* cl)
 367 {
 368     std::auto_ptr<HTTPRepository> repo;
 369     SGPath p(simgear::Dir::current().path());
 370     p.append("http_repo_basic");
 371     simgear::Dir pd(p);
 372     pd.removeChildren();
 373
 374     repo.reset(new HTTPRepository(p, cl));
 375     repo->setBaseUrl("http://localhost:2000/repo");
 376     repo->update();
 377
 378     waitForUpdateComplete(cl, repo.get());
 379
 380     verifyFileState(p, "fileA");
 381     verifyFileState(p, "dirA/subdirA/fileAAA");
 382     verifyFileState(p, "dirC/subdirA/subsubA/fileCAAA");
 383
 384     global_repo->findEntry("fileA")->revision++;
 385     global_repo->findEntry("dirB/subdirA/fileBAA")->revision++;
 386     global_repo->defineFile("dirC/fileCA"); // new file
 387     global_repo->findEntry("dirB/subdirA")->removeChild("fileBAB");
 388     global_repo->findEntry("dirA")->removeChild("subdirA"); // remove a dir
 389
 390     repo->update();
 391
 392     // verify deltas
 393     waitForUpdateComplete(cl, repo.get());
 394
 395     verifyFileState(p, "fileA");
 396     verifyFileState(p, "dirC/fileCA");
 397
 398     std::cout << "Passed test: basic clone and update" << std::endl;
 399 }
 400
 401 void testModifyLocalFiles(HTTP::Client* cl)
 402 {
 403     std::auto_ptr<HTTPRepository> repo;
 404     SGPath p(simgear::Dir::current().path());
 405     p.append("http_repo_modify_local_2");
 406     simgear::Dir pd(p);
 407     if (pd.exists()) {
 408         pd.removeChildren();
 409     }
 410
 411     repo.reset(new HTTPRepository(p, cl));
 412     repo->setBaseUrl("http://localhost:2000/repo");
 413     repo->update();
 414
 415     waitForUpdateComplete(cl, repo.get());
 416     verifyFileState(p, "dirB/subdirA/fileBAA");
 417
 418     SGPath modFile(p);
 419     modFile.append("dirB/subdirA/fileBAA");
 420     {
 421         std::ofstream of(modFile.c_str(), std::ios::out | std::ios::trunc);
 422         of << "complete nonsense";
 423         of.close();
 424     }
 425
 426     global_repo->clearRequestCounts();
 427     repo->update();
 428     waitForUpdateComplete(cl, repo.get());
 429     verifyFileState(p, "dirB/subdirA/fileBAA");
 430     verifyRequestCount("dirB", 0);
 431     verifyRequestCount("dirB/subdirA", 0);
 432     verifyRequestCount("dirB/subdirA/fileBAA", 1);
 433
 434     std::cout << "Passed test: identify and fix locally modified files" << std::endl;
 435 }
 436
 437 void testNoChangesUpdate()
 438 {
 439
 440 }
 441
 442 void testMergeExistingFileWithoutDownload(HTTP::Client* cl)
 443 {
 444     std::auto_ptr<HTTPRepository> repo;
 445     SGPath p(simgear::Dir::current().path());
 446     p.append("http_repo_merge_existing");
 447     simgear::Dir pd(p);
 448     if (pd.exists()) {
 449         pd.removeChildren();
 450     }
 451
 452     repo.reset(new HTTPRepository(p, cl));
 453     repo->setBaseUrl("http://localhost:2000/repo");
 454
 455     createFile(p, "dirC/fileCB", 4); // should match
 456     createFile(p, "dirC/fileCC", 3); // mismatch
 457
 458     global_repo->defineFile("dirC/fileCB", 4);
 459     global_repo->defineFile("dirC/fileCC", 10);
 460
 461     // new sub-tree
 462     createFile(p, "dirD/fileDA", 4);
 463     createFile(p, "dirD/subdirDA/fileDAA", 6);
 464     createFile(p, "dirD/subdirDB/fileDBA", 6);
 465
 466     global_repo->defineFile("dirD/fileDA", 4);
 467     global_repo->defineFile("dirD/subdirDA/fileDAA", 6);
 468     global_repo->defineFile("dirD/subdirDB/fileDBA", 6);
 469
 470     repo->update();
 471     waitForUpdateComplete(cl, repo.get());
 472     verifyFileState(p, "dirC/fileCB");
 473     verifyFileState(p, "dirC/fileCC");
 474     verifyRequestCount("dirC/fileCB", 0);
 475     verifyRequestCount("dirC/fileCC", 1);
 476
 477     verifyRequestCount("dirD/fileDA", 0);
 478     verifyRequestCount("dirD/subdirDA/fileDAA", 0);
 479     verifyRequestCount("dirD/subdirDB/fileDBA", 0);
 480
 481     std::cout << "Passed test: merge existing files with matching hash" << std::endl;
 482 }
 483
 484 void testLossOfLocalFiles(HTTP::Client* cl)
 485 {
 486     std::auto_ptr<HTTPRepository> repo;
 487     SGPath p(simgear::Dir::current().path());
 488     p.append("http_repo_lose_local");
 489     simgear::Dir pd(p);
 490     if (pd.exists()) {
 491         pd.removeChildren();
 492     }
 493
 494     repo.reset(new HTTPRepository(p, cl));
 495     repo->setBaseUrl("http://localhost:2000/repo");
 496     repo->update();
 497     waitForUpdateComplete(cl, repo.get());
 498     verifyFileState(p, "dirB/subdirA/fileBAA");
 499
 500     SGPath lostPath(p);
 501     lostPath.append("dirB/subdirA");
 502     simgear::Dir lpd(lostPath);
 503     lpd.remove(true);
 504
 505     global_repo->clearRequestCounts();
 506
 507     repo->update();
 508     waitForUpdateComplete(cl, repo.get());
 509     verifyFileState(p, "dirB/subdirA/fileBAA");
 510
 511     verifyRequestCount("dirB", 0);
 512     verifyRequestCount("dirB/subdirA", 1);
 513     verifyRequestCount("dirB/subdirA/fileBAC", 1);
 514
 515     std::cout << "Passed test: lose and replace local files" << std::endl;
 516 }
 517
 518 void testAbandonMissingFiles(HTTP::Client* cl)
 519 {
 520     std::auto_ptr<HTTPRepository> repo;
 521     SGPath p(simgear::Dir::current().path());
 522     p.append("http_repo_missing_files");
 523     simgear::Dir pd(p);
 524     if (pd.exists()) {
 525         pd.removeChildren();
 526     }
 527
 528     global_repo->defineFile("dirA/subdirE/fileAEA");
 529     global_repo->findEntry("dirA/subdirE/fileAEA")->setGetWillFail(true);
 530
 531     repo.reset(new HTTPRepository(p, cl));
 532     repo->setBaseUrl("http://localhost:2000/repo");
 533     repo->update();
 534     waitForUpdateComplete(cl, repo.get());
 535     if (repo->failure() != AbstractRepository::REPO_PARTIAL_UPDATE) {
 536         throw sg_exception("Bad result from missing files test");
 537     }
 538
 539     global_repo->findEntry("dirA/subdirE/fileAEA")->setGetWillFail(false);
 540 }
 541
 542 void testAbandonCorruptFiles(HTTP::Client* cl)
 543 {
 544     std::auto_ptr<HTTPRepository> repo;
 545     SGPath p(simgear::Dir::current().path());
 546     p.append("http_repo_corrupt_files");
 547     simgear::Dir pd(p);
 548     if (pd.exists()) {
 549         pd.removeChildren();
 550     }
 551
 552     global_repo->defineFile("dirB/subdirG/fileBGA");
 553     global_repo->findEntry("dirB/subdirG/fileBGA")->setReturnCorruptData(true);
 554
 555     repo.reset(new HTTPRepository(p, cl));
 556     repo->setBaseUrl("http://localhost:2000/repo");
 557     repo->update();
 558     waitForUpdateComplete(cl, repo.get());
 559     if (repo->failure() != AbstractRepository::REPO_PARTIAL_UPDATE) {
 560         throw sg_exception("Bad result from corrupt files test");
 561     }
 562
 563     std::cout << "Passed test: detect corrupted download" << std::endl;
 564 }
 565
 566 int main(int argc, char* argv[])
 567 {
 568   sglog().setLogLevels( SG_ALL, SG_INFO );
 569
 570   HTTP::Client cl;
 571   cl.setMaxConnections(1);
 572
 573     global_repo = new TestRepoEntry(NULL, "root", true);
 574     global_repo->defineFile("fileA");
 575     global_repo->defineFile("fileB");
 576     global_repo->defineFile("dirA/fileAA");
 577     global_repo->defineFile("dirA/fileAB");
 578     global_repo->defineFile("dirA/fileAC");
 579     global_repo->defineFile("dirA/subdirA/fileAAA");
 580     global_repo->defineFile("dirA/subdirA/fileAAB");
 581     global_repo->defineFile("dirB/subdirA/fileBAA");
 582     global_repo->defineFile("dirB/subdirA/fileBAB");
 583     global_repo->defineFile("dirB/subdirA/fileBAC");
 584     global_repo->defineFile("dirC/subdirA/subsubA/fileCAAA");
 585
 586     testBasicClone(&cl);
 587
 588     testModifyLocalFiles(&cl);
 589
 590     testLossOfLocalFiles(&cl);
 591
 592     testMergeExistingFileWithoutDownload(&cl);
 593
 594     testAbandonMissingFiles(&cl);
 595
 596     testAbandonCorruptFiles(&cl);
 597
 598     return 0;
 599 }