bz2 support for OSM reading and writing

CMakeLists.txt

@@ -49,6 +49,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPFAEDLE_PRECISION=${PFAEDLE_PRECISION}
 
 find_package(LibZip)
 find_package(ZLIB)
+find_package(BZip2)
 
 if (LIBZIP_FOUND)
 	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLIBZIP_FOUND=1")
@@ -60,6 +61,12 @@ else ()
 	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPFXML_NO_ZLIB=1")
 endif()
 
+if (BZIP2_FOUND)
+	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBZLIB_FOUND=1")
+else ()
+	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPFXML_NO_BZLIB=1")
+endif()
+
 # http://brianmilco.blogspot.de/2012/11/cmake-automatically-use-git-tags-as.html
 include(GetGitRevisionDescription)
 git_get_tag(VERSION_GIT)

README.md

@@ -19,6 +19,7 @@ For a quick visual inspection of the shape quality, see for example the schedule
  * `gcc >= 5.0` (or `clang >= 3.9`)
  * `libzip` (optional, for ZIP support)
  * `zlib` (optional, for gzip support)
+ * `libbz2` (optional, for bzip2 support)
 
 ## Building and Installation
 

src/pfaedle/CMakeLists.txt

@@ -6,6 +6,7 @@ list(REMOVE_ITEM pfaedle_SRC ${pfaedle_main})
 
 include_directories(
 	${PFAEDLE_INCLUDE_DIR}
+	SYSTEM ${BZIP2_INCLUDE_DIR}
 	SYSTEM ${LIBZIP_INCLUDE_DIR}
 	SYSTEM ${LIBZIP_CONF_INCLUDE_DIR}
 )
@@ -19,6 +20,6 @@ add_executable(pfaedle ${pfaedle_main})
 add_library(pfaedle_dep ${pfaedle_SRC})
 
 include_directories(pfaedle_dep PUBLIC ${PROJECT_SOURCE_DIR}/src/xml/include/ ${PROJECT_SOURCE_DIR}/src/cppgtfs/src)
-target_link_libraries(pfaedle pfaedle_dep util configparser ad_cppgtfs -lpthread ${LIBZIP_LIBRARY})
+target_link_libraries(pfaedle pfaedle_dep util configparser ad_cppgtfs -lpthread ${LIBZIP_LIBRARY} ${BZIP2_LIBRARIES})
 
 add_subdirectory(tests)

src/pfaedle/osm/OsmBuilder.cpp

@@ -379,7 +379,7 @@ void OsmBuilder::filterWrite(const std::string& in, const std::string& out,
     latLngBox.add(Box<double>({minlon, minlat}, {maxlon, maxlat}));
   }
 
-  util::xml::XmlWriter wr(out, true, 4);
+  util::xml::XmlWriter wr(out, false, 0);
 
   wr.put("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
   wr.openTag("osm", {{"version", "0.6"},

src/util/CMakeLists.txt

@@ -2,8 +2,20 @@ file(GLOB_RECURSE util_SRC *.cpp)
 list(REMOVE_ITEM util_SRC TestMain.cpp)
 add_library(util ${util_SRC})
 
+if (!BZIP2_FOUND)
+    find_package(BZip2)
+    if (ZLIB_FOUND)
+		add_definitions( -DBZLIB_FOUND=${BZIP2_FOUND} )
+	endif()
+endif()
+
+if (BZIP2_FOUND)
+	include_directories( ${BZIP2_INCLUDE_DIR} )
+	target_link_libraries( util ${BZIP2_LIBRARIES} )
+endif(BZIP2_FOUND)
+
 if (!ZLIB_FOUND)
-    find_package( ZLIB )
+    find_package(ZLIB)
     if (ZLIB_FOUND)
 		add_definitions( -DZLIB_FOUND=${ZLIB_FOUND} )
 	endif()
@@ -12,6 +24,6 @@ endif()
 if (ZLIB_FOUND)
 	include_directories( ${ZLIB_INCLUDE_DIRS} )
 	target_link_libraries( util ${ZLIB_LIBRARIES} )
-endif( ZLIB_FOUND )
+endif(ZLIB_FOUND)
 
 add_subdirectory(tests)

src/util/xml/XmlWriter.cpp

@@ -5,9 +5,13 @@
 #include <algorithm>
 #include <fstream>
 #include <map>
+#include <cstring>
 #include <ostream>
 #include <stack>
 #include <string>
+#ifdef BZLIB_FOUND
+#include <bzlib.h>
+#endif
 
 #include "XmlWriter.h"
 
@@ -20,15 +24,15 @@ using std::string;
 
 // _____________________________________________________________________________
 XmlWriter::XmlWriter(std::ostream* out)
-    : _out(out), _pretty(false), _indent(4), _gzfile(0) {}
+    : _out(out), _pretty(false), _indent(4), _gzfile(0), _bzfile(0) {}
 
 // _____________________________________________________________________________
 XmlWriter::XmlWriter(std::ostream* out, bool pret)
-    : _out(out), _pretty(pret), _indent(4), _gzfile(0) {}
+    : _out(out), _pretty(pret), _indent(4), _gzfile(0), _bzfile(0) {}
 
 // _____________________________________________________________________________
 XmlWriter::XmlWriter(std::ostream* out, bool pret, size_t indent)
-    : _out(out), _pretty(pret), _indent(indent), _gzfile(0) {}
+    : _out(out), _pretty(pret), _indent(indent), _gzfile(0), _bzfile(0) {}
 
 // _____________________________________________________________________________
 XmlWriter::XmlWriter(const std::string& file)
@@ -40,7 +44,7 @@ XmlWriter::XmlWriter(const std::string& file, bool pret)
 
 // _____________________________________________________________________________
 XmlWriter::XmlWriter(const std::string& file, bool pret, size_t indent)
-    : _out(0), _pretty(pret), _indent(indent), _gzfile(0) {
+    : _out(0), _pretty(pret), _indent(indent), _gzfile(0), _bzfile(0) {
   if (file.size() > 2 && file[file.size() - 1] == 'z' &&
       file[file.size() - 2] == 'g' && file[file.size() - 3] == '.') {
 #ifdef ZLIB_FOUND
@@ -52,6 +56,26 @@ XmlWriter::XmlWriter(const std::string& file, bool pret, size_t indent)
     throw std::runtime_error(
         "Could not open gzip file for writing, was compiled without gzip "
         "support");
+#endif
+  } else if (file.size() > 3 && file[file.size() - 1] == '2' &&
+             file[file.size() - 2] == 'z' && file[file.size() - 3] == 'b' &&
+             file[file.size() - 4] == '.') {
+#ifdef BZLIB_FOUND
+    _bzbuf = new char[BUFFER_S];
+
+    FILE* f = fopen(file.c_str(), "w");
+    int err;
+    if (!f) throw std::runtime_error("Could not open file for writing.");
+
+    _bzfile = BZ2_bzWriteOpen(&err, f, 9, 0, 30);
+
+    if (err != BZ_OK) {
+      throw std::runtime_error("Could not open bzip file for writing.");
+    }
+#else
+    throw std::runtime_error(
+        "Could not open bzip file for writing, was compiled without bzip "
+        "support");
 #endif
   } else {
     _outs.open(file);
@@ -176,18 +200,43 @@ void XmlWriter::put(const string& str) {
   if (_gzfile) {
 #ifdef ZLIB_FOUND
     gzwrite(_gzfile, str.c_str(), str.size());
+#endif
+  } else if (_bzfile) {
+#ifdef BZLIB_FOUND
+    if (_bzbufpos == BUFFER_S || _bzbufpos + str.size() > BUFFER_S) flushBzip();
+    memcpy( _bzbuf + _bzbufpos, str.c_str(), str.size());
+    _bzbufpos += str.size();
 #endif
   } else {
     _out->write(str.c_str(), str.size());
   }
 }
 
+// _____________________________________________________________________________
+void XmlWriter::flushBzip() {
+#ifdef BZLIB_FOUND
+    int err = 0;
+    BZ2_bzWrite(&err, _bzfile, _bzbuf, _bzbufpos);
+    if (err == BZ_IO_ERROR) {
+      BZ2_bzWriteClose(&err, _bzfile, 0, 0, 0);
+      throw std::runtime_error("Could not write to file.");
+    }
+
+    _bzbufpos = 0;
+  #endif
+}
+
 // _____________________________________________________________________________
 void XmlWriter::put(const char c) {
   if (_gzfile) {
 #ifdef ZLIB_FOUND
     gzputc(_gzfile, c);
 
+#endif
+  } else if (_bzfile) {
+#ifdef BZLIB_FOUND
+    _bzbuf[_bzbufpos++] = c;
+    if (_bzbufpos == BUFFER_S) flushBzip();
 #endif
   } else {
     _out->put(c);

src/util/xml/XmlWriter.h

@@ -8,6 +8,9 @@
 #ifdef ZLIB_FOUND
 #include <zlib.h>
 #endif
+#ifdef BZLIB_FOUND
+#include <bzlib.h>
+#endif
 #include <map>
 #include <ostream>
 #include <fstream>
@@ -17,6 +20,8 @@
 namespace util {
 namespace xml {
 
+static const size_t BUFFER_S = 32 * 1024 * 1024;
+
 class XmlWriterException : public std::exception {
  public:
   XmlWriterException(std::string msg) : _msg(msg) {}
@@ -41,6 +46,13 @@ class XmlWriter {
   ~XmlWriter(){
 #ifdef ZLIB_FOUND
     if (_gzfile) gzclose(_gzfile);
+#endif
+#ifdef BZLIB_FOUND
+    int err;
+    if (_bzfile) {
+      flushBzip();
+      BZ2_bzWriteClose(&err, _bzfile, 0, 0, 0);
+    }
 #endif
   };
 
@@ -83,6 +95,8 @@ class XmlWriter {
     bool hanging;
   };
 
+  void flushBzip();
+
   std::ostream* _out;
   std::ofstream _outs;
   std::stack<XmlNode> _nstack;
@@ -96,6 +110,14 @@ class XmlWriter {
   int _gzfile;
 #endif
 
+  char* _bzbuf;
+  size_t _bzbufpos = 0;
+#ifdef BZLIB_FOUND
+  BZFILE* _bzfile;
+#else
+  int _bzfile;
+#endif
+
   // handles indentation
   void doIndent();