From 5f2a2a7862e8f3c7f0da2d64545726058f620cc7 Mon Sep 17 00:00:00 2001 From: James Turner Date: Wed, 4 May 2016 20:35:48 +0100 Subject: [PATCH] Initial Tar package support. Needs proper testing, but basic unit-test passes. --- simgear/package/CMakeLists.txt | 9 +- simgear/package/CatalogTest.cxx | 42 ++- simgear/package/Install.cxx | 28 +- simgear/package/catalogTest1/b737.tar.gz | Bin 0 -> 347 bytes .../catalogTest1/b737NG/b737-700-set.xml | 7 + .../catalogTest1/b737NG/b737-800-set.xml | 7 + .../catalogTest1/b737NG/b737-900-set.xml | 7 + .../catalogTest1/b737NG/b737-common-set.xml | 9 + simgear/package/catalogTest1/catalog-v2.xml | 24 ++ simgear/package/catalogTest1/catalog.xml | 23 ++ simgear/package/untar.cxx | 341 ++++++++++++++++++ simgear/package/untar.hxx | 52 +++ 12 files changed, 541 insertions(+), 8 deletions(-) create mode 100644 simgear/package/catalogTest1/b737.tar.gz create mode 100644 simgear/package/catalogTest1/b737NG/b737-700-set.xml create mode 100644 simgear/package/catalogTest1/b737NG/b737-800-set.xml create mode 100644 simgear/package/catalogTest1/b737NG/b737-900-set.xml create mode 100644 simgear/package/catalogTest1/b737NG/b737-common-set.xml create mode 100644 simgear/package/untar.cxx create mode 100644 simgear/package/untar.hxx diff --git a/simgear/package/CMakeLists.txt b/simgear/package/CMakeLists.txt index 37e7a504..a031edaa 100644 --- a/simgear/package/CMakeLists.txt +++ b/simgear/package/CMakeLists.txt @@ -1,7 +1,7 @@ include (SimGearComponent) -set(HEADERS +set(HEADERS Catalog.hxx Package.hxx Install.hxx @@ -9,7 +9,7 @@ set(HEADERS Delegate.hxx ) -set(SOURCES +set(SOURCES Catalog.cxx Package.cxx Install.cxx @@ -18,6 +18,7 @@ set(SOURCES md5.h md5.c ioapi.c ioapi_mem.c ioapi.h unzip.h unzip.c + untar.hxx untar.cxx ) simgear_component(package package "${SOURCES}" "${HEADERS}") @@ -34,7 +35,7 @@ target_link_libraries(catalog_test ${TEST_LIBS}) set_target_properties(catalog_test PROPERTIES COMPILE_DEFINITIONS "SRC_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}\"" ) - + add_test(catalog_test ${EXECUTABLE_OUTPUT_PATH}/catalog_test) -endif(ENABLE_TESTS) \ No newline at end of file +endif(ENABLE_TESTS) diff --git a/simgear/package/CatalogTest.cxx b/simgear/package/CatalogTest.cxx index 22a4cd54..02c97c19 100644 --- a/simgear/package/CatalogTest.cxx +++ b/simgear/package/CatalogTest.cxx @@ -133,7 +133,7 @@ int parseTest() COMPARE(cat->description(), "First test catalog"); // check the packages too - COMPARE(cat->packages().size(), 3); + COMPARE(cat->packages().size(), 4); pkg::PackageRef p1 = cat->packages().front(); COMPARE(p1->catalog(), cat.ptr()); @@ -208,7 +208,7 @@ void testAddCatalog(HTTP::Client* cl) p.append("org.flightgear.test.catalog1"); p.append("catalog.xml"); VERIFY(p.exists()); - COMPARE(root->allPackages().size(), 3); + COMPARE(root->allPackages().size(), 4); COMPARE(root->catalogs().size(), 1); pkg::PackageRef p1 = root->getPackageById("alpha"); @@ -403,6 +403,42 @@ void testRefreshCatalog(HTTP::Client* cl) COMPARE(root->getPackageById("common-sounds")->revision(), 11); } +void testInstallTarPackage(HTTP::Client* cl) +{ + SGPath rootPath(simgear::Dir::current().path()); + rootPath.append("pkg_install_tar"); + simgear::Dir pd(rootPath); + pd.removeChildren(); + + pkg::RootRef root(new pkg::Root(rootPath, "8.1.2")); + // specify a test dir + root->setHTTPClient(cl); + + pkg::CatalogRef c = pkg::Catalog::createFromUrl(root.ptr(), "http://localhost:2000/catalogTest1/catalog.xml"); + waitForUpdateComplete(cl, root); + + pkg::PackageRef p1 = root->getPackageById("org.flightgear.test.catalog1.b737-NG"); + COMPARE(p1->id(), "b737-NG"); + pkg::InstallRef ins = p1->install(); + + VERIFY(ins->isQueued()); + + waitForUpdateComplete(cl, root); + VERIFY(p1->isInstalled()); + VERIFY(p1->existingInstall() == ins); + + // verify on disk state + SGPath p(rootPath); + p.append("org.flightgear.test.catalog1"); + p.append("Aircraft"); + p.append("b737NG"); + + COMPARE(p, ins->path()); + + p.append("b737-900-set.xml"); + VERIFY(p.exists()); +} + int main(int argc, char* argv[]) { @@ -425,6 +461,8 @@ int main(int argc, char* argv[]) testRefreshCatalog(&cl); + testInstallTarPackage(&cl); + std::cout << "Successfully passed all tests!" << std::endl; return EXIT_SUCCESS; } diff --git a/simgear/package/Install.cxx b/simgear/package/Install.cxx index cdfc7103..d43dff92 100644 --- a/simgear/package/Install.cxx +++ b/simgear/package/Install.cxx @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -144,8 +145,8 @@ protected: return; } - if (!extractUnzip()) { - SG_LOG(SG_GENERAL, SG_WARN, "zip extraction failed"); + if (!extract()) { + SG_LOG(SG_GENERAL, SG_WARN, "archive extraction failed"); doFailure(Delegate::FAIL_EXTRACT); return; } @@ -250,6 +251,22 @@ private: unzCloseCurrentFile(zip); } + bool extract() + { + const std::string u(url()); + const size_t ul(u.length()); + if (u.rfind(".zip") == (ul - 4)) { + return extractUnzip(); + } + + if (u.rfind(".tar.gz") == (ul - 7)) { + return extractTar(); + } + + SG_LOG(SG_IO, SG_WARN, "unsupported archive format:" << u); + return false; + } + bool extractUnzip() { bool result = true; @@ -287,6 +304,13 @@ private: return result; } + bool extractTar() + { + TarExtractor tx(m_extractPath); + tx.extractBytes(m_buffer.data(), m_buffer.size()); + return !tx.hasError() && tx.isAtEndOfArchive(); + } + void doFailure(Delegate::StatusCode aReason) { Dir dir(m_extractPath); diff --git a/simgear/package/catalogTest1/b737.tar.gz b/simgear/package/catalogTest1/b737.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..05e452bd132240a6839d84738c10d5908b5439c1 GIT binary patch literal 347 zcmV-h0i^yPiwFRqb3<1E1MSyAZ-Ouw24K$oipIOrmX>P5Fxh37W!Ya4hlXWs^~wZMX7Tn3MDDw9V$k_4YmvbtF>P|E5e8rBj`m*=OwDPGBNF0x|CxQ# z^n@JC@&3n4+imiZ)jF@ry9t}o$ujZ%__?ljS#7=^b8QlzkXYwM zQ|;&_)9X6l8M`34ud=+{lJ?sZ2i2Gl~2U;UD=o ttF3lJ_aI64hd>-OV>c!EnS%oW0000000000007_uz5&nMYMTHk000byw@m;5 literal 0 HcmV?d00001 diff --git a/simgear/package/catalogTest1/b737NG/b737-700-set.xml b/simgear/package/catalogTest1/b737NG/b737-700-set.xml new file mode 100644 index 00000000..33d5a640 --- /dev/null +++ b/simgear/package/catalogTest1/b737NG/b737-700-set.xml @@ -0,0 +1,7 @@ + + + + + Boeing 737-700 + + diff --git a/simgear/package/catalogTest1/b737NG/b737-800-set.xml b/simgear/package/catalogTest1/b737NG/b737-800-set.xml new file mode 100644 index 00000000..760410f0 --- /dev/null +++ b/simgear/package/catalogTest1/b737NG/b737-800-set.xml @@ -0,0 +1,7 @@ + + + + + Boeing 737-800 + + diff --git a/simgear/package/catalogTest1/b737NG/b737-900-set.xml b/simgear/package/catalogTest1/b737NG/b737-900-set.xml new file mode 100644 index 00000000..b249069c --- /dev/null +++ b/simgear/package/catalogTest1/b737NG/b737-900-set.xml @@ -0,0 +1,7 @@ + + + + + Boeing 737-900 + + diff --git a/simgear/package/catalogTest1/b737NG/b737-common-set.xml b/simgear/package/catalogTest1/b737NG/b737-common-set.xml new file mode 100644 index 00000000..5d60aaf1 --- /dev/null +++ b/simgear/package/catalogTest1/b737NG/b737-common-set.xml @@ -0,0 +1,9 @@ + + + + + + boeing + + + diff --git a/simgear/package/catalogTest1/catalog-v2.xml b/simgear/package/catalogTest1/catalog-v2.xml index 6749b000..a66ac7a5 100644 --- a/simgear/package/catalogTest1/catalog-v2.xml +++ b/simgear/package/catalogTest1/catalog-v2.xml @@ -78,6 +78,30 @@ acf9eb89cf396eb42f8823d9cdf17584 + + b737-NG + Boeing 737 NG + b737NG + A popular twin-engined narrow body jet + 112 + 860 + + boeing + jet + ifr + + + 5 + 5 + 4 + 4 + + + a94ca5704f305b90767f40617d194ed6 + http://localhost:2000/catalogTest1/b737.tar.gz + + + dc3 DC-3 diff --git a/simgear/package/catalogTest1/catalog.xml b/simgear/package/catalogTest1/catalog.xml index 6f0504f6..5da1400f 100644 --- a/simgear/package/catalogTest1/catalog.xml +++ b/simgear/package/catalogTest1/catalog.xml @@ -68,6 +68,29 @@ + + b737-NG + Boeing 737 NG + b737NG + A popular twin-engined narrow body jet + 111 + 860 + + boeing + jet + ifr + + + 5 + 5 + 4 + 4 + + + a94ca5704f305b90767f40617d194ed6 + http://localhost:2000/catalogTest1/b737.tar.gz + + common-sounds Common sound files for test catalog aircraft diff --git a/simgear/package/untar.cxx b/simgear/package/untar.cxx new file mode 100644 index 00000000..715d2243 --- /dev/null +++ b/simgear/package/untar.cxx @@ -0,0 +1,341 @@ +// Copyright (C) 2016 James Turner - +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Library General Public +// License as published by the Free Software Foundation; either +// version 2 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +// + +#include "untar.hxx" + +#include +#include + +#include + +#include +#include + +#include + +namespace simgear +{ + +namespace pkg +{ + + const int ZLIB_DECOMPRESS_BUFFER_SIZE = 32 * 1024; + const int ZLIB_INFLATE_WINDOW_BITS = MAX_WBITS; + const int ZLIB_DECODE_GZIP_HEADER = 16; + +/* tar Header Block, from POSIX 1003.1-1990. */ + +typedef struct +{ + char fileName[100]; + char mode[8]; /* 100 */ + char uid[8]; /* 108 */ + char gid[8]; /* 116 */ + char size[12]; /* 124 */ + char mtime[12]; /* 136 */ + char chksum[8]; /* 148 */ + char typeflag; /* 156 */ + char linkname[100]; /* 157 */ + char magic[6]; /* 257 */ + char version[2]; /* 263 */ + char uname[32]; /* 265 */ + char gname[32]; /* 297 */ + char devmajor[8]; /* 329 */ + char devminor[8]; /* 337 */ + char prefix[155]; /* 345 */ +} UstarHeaderBlock; + + const size_t TAR_HEADER_BLOCK_SIZE = 512; + +#define TMAGIC "ustar" /* ustar and a null */ +#define TMAGLEN 6 +#define TVERSION "00" /* 00 and no null */ +#define TVERSLEN 2 + + /* Values used in typeflag field. */ +#define REGTYPE '0' /* regular file */ +#define AREGTYPE '\0' /* regular file */ +#define LNKTYPE '1' /* link */ +#define SYMTYPE '2' /* reserved */ +#define CHRTYPE '3' /* character special */ +#define BLKTYPE '4' /* block special */ +#define DIRTYPE '5' /* directory */ +#define FIFOTYPE '6' /* FIFO special */ +#define CONTTYPE '7' /* reserved */ + +class TarExtractorPrivate +{ +public: + typedef enum { + INVALID = 0, + READING_HEADER, + READING_FILE, + READING_PADDING, + PRE_END_OF_ARCHVE, + END_OF_ARCHIVE, + ERROR_STATE, ///< states above this are error conditions + BAD_ARCHIVE, + BAD_DATA + } State; + + SGPath path; + State state; + union { + UstarHeaderBlock header; + uint8_t headerBytes[TAR_HEADER_BLOCK_SIZE]; + }; + + size_t bytesRemaining; + std::auto_ptr currentFile; + size_t currentFileSize; + z_stream zlibStream; + uint8_t* zlibOutput; + bool haveInitedZLib; + uint8_t* headerPtr; + + TarExtractorPrivate() : + haveInitedZLib(false) + { + } + + ~TarExtractorPrivate() + { + free(zlibOutput); + } + + void checkEndOfState() + { + if (bytesRemaining > 0) { + return; + } + + if (state == READING_FILE) { + currentFile->close(); + size_t pad = currentFileSize % TAR_HEADER_BLOCK_SIZE; + if (pad) { + bytesRemaining = TAR_HEADER_BLOCK_SIZE - pad; + setState(READING_PADDING); + } else { + setState(READING_HEADER); + } + } else if (state == READING_HEADER) { + processHeader(); + } else if (state == PRE_END_OF_ARCHVE) { + if (headerIsAllZeros()) { + setState(END_OF_ARCHIVE); + } else { + // what does the spec say here? + } + } else if (state == READING_PADDING) { + setState(READING_HEADER); + } + } + + void setState(State newState) + { + if ((newState == READING_HEADER) || (newState == PRE_END_OF_ARCHVE)) { + bytesRemaining = TAR_HEADER_BLOCK_SIZE; + headerPtr = headerBytes; + } + + state = newState; + } + + void processHeader() + { + if (headerIsAllZeros()) { + if (state == PRE_END_OF_ARCHVE) { + setState(END_OF_ARCHIVE); + } else { + setState(PRE_END_OF_ARCHVE); + } + return; + } + + if (strncmp(header.magic, TMAGIC, TMAGLEN) != 0) { + SG_LOG(SG_IO, SG_WARN, "magic is wrong"); + state = BAD_ARCHIVE; + return; + } + + std::string tarPath = std::string(header.prefix) + std::string(header.fileName); + + if (!isSafePath(tarPath)) { + //state = BAD_ARCHIVE; + SG_LOG(SG_IO, SG_WARN, "bad tar path:" << tarPath); + //return; + } + + SGPath p = path; + p.append(tarPath); + + if (header.typeflag == DIRTYPE) { + Dir dir(p); + dir.create(0755); + setState(READING_HEADER); + } else if ((header.typeflag == REGTYPE) || (header.typeflag == AREGTYPE)) { + currentFileSize = ::strtol(header.size, NULL, 8); + bytesRemaining = currentFileSize; + currentFile.reset(new SGBinaryFile(p.str())); + currentFile->open(SG_IO_OUT); + setState(READING_FILE); + } else { + SG_LOG(SG_IO, SG_WARN, "Unsupported tar file type:" << header.typeflag); + state = BAD_ARCHIVE; + } + } + + void processBytes(const char* bytes, size_t count) + { + if ((state >= ERROR_STATE) || (state == END_OF_ARCHIVE)) { + return; + } + + size_t curBytes = std::min(bytesRemaining, count); + if (state == READING_FILE) { + currentFile->write(bytes, curBytes); + bytesRemaining -= curBytes; + } else if ((state == READING_HEADER) || (state == PRE_END_OF_ARCHVE) || (state == END_OF_ARCHIVE)) { + memcpy(headerPtr, bytes, curBytes); + bytesRemaining -= curBytes; + headerPtr += curBytes; + } else if (state == READING_PADDING) { + bytesRemaining -= curBytes; + } + + checkEndOfState(); + if (count > curBytes) { + // recurse with unprocessed bytes + processBytes(bytes + curBytes, count - curBytes); + } + } + + bool headerIsAllZeros() const + { + char* headerAsChar = (char*) &header; + for (size_t i=0; i < offsetof(UstarHeaderBlock, magic); ++i) { + if (*headerAsChar++ != 0) { + return false; + } + } + + return true; + } + + bool isSafePath(const std::string& p) const + { + if (p.empty()) { + return false; + } + + // reject absolute paths + if (p.front() == '/') { + return false; + } + + // reject paths containing '..' + size_t doubleDot = p.find(".."); + if (doubleDot != std::string::npos) { + return false; + } + + // on POSIX could use realpath to sanity check + return true; + } +}; + +TarExtractor::TarExtractor(const SGPath& rootPath) : + d(new TarExtractorPrivate) +{ + + d->path = rootPath; + d->state = TarExtractorPrivate::INVALID; + + memset(&d->zlibStream, 0, sizeof(z_stream)); + d->zlibOutput = (unsigned char*) malloc(ZLIB_DECOMPRESS_BUFFER_SIZE); + d->zlibStream.zalloc = Z_NULL; + d->zlibStream.zfree = Z_NULL; + + d->zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE; + d->zlibStream.next_out = d->zlibOutput; +} + +void TarExtractor::extractBytes(const char* bytes, size_t count) +{ + if (d->state >= TarExtractorPrivate::ERROR_STATE) { + return; + } + + d->zlibStream.next_in = (uint8_t*) bytes; + d->zlibStream.avail_in = count; + + if (!d->haveInitedZLib) { + if (inflateInit2(&d->zlibStream, ZLIB_INFLATE_WINDOW_BITS | ZLIB_DECODE_GZIP_HEADER) != Z_OK) { + SG_LOG(SG_IO, SG_WARN, "inflateInit2 failed"); + d->state = TarExtractorPrivate::BAD_DATA; + return; + } else { + d->haveInitedZLib = true; + d->setState(TarExtractorPrivate::READING_HEADER); + } + } + + size_t writtenSize; + + // loop, running zlib() inflate and sending output bytes to + // our request body handler. Keep calling inflate until no bytes are + // written, and ZLIB has consumed all available input + do { + d->zlibStream.next_out = d->zlibOutput; + d->zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE; + int result = inflate(&d->zlibStream, Z_NO_FLUSH); + if (result == Z_OK || result == Z_STREAM_END) { + // nothing to do + + } else if (result == Z_BUF_ERROR) { + // transient error, fall through + } else { + // _error = result; + SG_LOG(SG_IO, SG_WARN, "Permanent ZLib error:" << d->zlibStream.msg); + d->state = TarExtractorPrivate::BAD_DATA; + return; + } + + writtenSize = ZLIB_DECOMPRESS_BUFFER_SIZE - d->zlibStream.avail_out; + if (writtenSize > 0) { + d->processBytes((const char*) d->zlibOutput, writtenSize); + } + + if (result == Z_STREAM_END) { + break; + } + } while ((d->zlibStream.avail_in > 0) || (writtenSize > 0)); +} + +bool TarExtractor::isAtEndOfArchive() const +{ + return (d->state == TarExtractorPrivate::END_OF_ARCHIVE); +} + +bool TarExtractor::hasError() const +{ + return (d->state >= TarExtractorPrivate::ERROR_STATE); +} + +} // of pkg + +} // of simgear diff --git a/simgear/package/untar.hxx b/simgear/package/untar.hxx new file mode 100644 index 00000000..76cc1f9d --- /dev/null +++ b/simgear/package/untar.hxx @@ -0,0 +1,52 @@ +// Copyright (C) 2016 James Turner - +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Library General Public +// License as published by the Free Software Foundation; either +// version 2 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +// + +#ifndef SG_PKG_UNTAR_HXX +#define SG_PKG_UNTAR_HXX + +#include + +#include + +namespace simgear +{ + +namespace pkg +{ + +class TarExtractorPrivate; + +class TarExtractor +{ +public: + TarExtractor(const SGPath& rootPath); + + void extractBytes(const char* bytes, size_t count); + + bool isAtEndOfArchive() const; + + bool hasError() const; + +private: + std::auto_ptr d; +}; + +} // of namespace pkg + +} // of namespace simgear + +#endif -- 2.39.5