Libosmium  2.15.2
Fast and flexible C++ library for working with OpenStreetMap data
bzip2_compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_BZIP2_COMPRESSION_HPP
2 #define OSMIUM_IO_BZIP2_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2019 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
46 #include <osmium/io/detail/read_write.hpp>
47 #include <osmium/io/error.hpp>
50 #include <osmium/util/file.hpp>
51 
52 #include <bzlib.h>
53 
54 #include <cassert>
55 #include <cerrno>
56 #include <cstdio>
57 #include <limits>
58 #include <string>
59 #include <system_error>
60 
61 #ifndef _MSC_VER
62 # include <unistd.h>
63 #endif
64 
65 namespace osmium {
66 
71  struct bzip2_error : public io_error {
72 
74  int system_errno = 0;
75 
76  bzip2_error(const std::string& what, const int error_code) :
77  io_error(what),
78  bzip2_error_code(error_code) {
79  if (error_code == BZ_IO_ERROR) {
80  system_errno = errno;
81  }
82  }
83 
84  }; // struct bzip2_error
85 
86  namespace io {
87 
88  namespace detail {
89 
90  [[noreturn]] inline void throw_bzip2_error(BZFILE* bzfile, const char* msg, const int bzlib_error) {
91  std::string error{"bzip2 error: "};
92  error += msg;
93  error += ": ";
94  int errnum = bzlib_error;
95  if (bzlib_error) {
96  error += std::to_string(bzlib_error);
97  } else if (bzfile) {
98  error += ::BZ2_bzerror(bzfile, &errnum);
99  }
100  throw osmium::bzip2_error{error, errnum};
101  }
102 
103  class file_wrapper {
104 
105  FILE* m_file = nullptr;
106 
107  public:
108 
109  file_wrapper() noexcept = default;
110 
111  file_wrapper(const int fd, const char* mode) {
112 #ifdef _MSC_VER
113  osmium::detail::disable_invalid_parameter_handler diph;
114 #endif
115  m_file = fdopen(fd, mode);
116  if (!m_file) {
117  ::close(fd);
118  throw std::system_error{errno, std::system_category(), "fdopen failed"};
119  }
120  }
121 
122  file_wrapper(const file_wrapper&) = delete;
123  file_wrapper& operator=(const file_wrapper&) = delete;
124 
125  file_wrapper(file_wrapper&&) = delete;
126  file_wrapper& operator=(file_wrapper&&) = delete;
127 
128  ~file_wrapper() noexcept {
129 #ifdef _MSC_VER
130  osmium::detail::disable_invalid_parameter_handler diph;
131 #endif
132  if (m_file) {
133  fclose(m_file);
134  }
135  }
136 
137  FILE* file() const noexcept {
138  return m_file;
139  }
140 
141  void close() {
142 #ifdef _MSC_VER
143  osmium::detail::disable_invalid_parameter_handler diph;
144 #endif
145  if (m_file) {
146  if (fclose(m_file) != 0) {
147  m_file = nullptr;
148  throw std::system_error{errno, std::system_category(), "fclose failed"};
149  }
150  m_file = nullptr;
151  }
152  }
153 
154  }; // class file_wrapper
155 
156  } // namespace detail
157 
158  class Bzip2Compressor : public Compressor {
159 
160  detail::file_wrapper m_file;
161  BZFILE* m_bzfile = nullptr;
162 
163  public:
164 
165  explicit Bzip2Compressor(const int fd, const fsync sync) :
166  Compressor(sync),
167  m_file(fd, "wb") {
168 #ifdef _MSC_VER
169  osmium::detail::disable_invalid_parameter_handler diph;
170 #endif
171  int bzerror = BZ_OK;
172  m_bzfile = ::BZ2_bzWriteOpen(&bzerror, m_file.file(), 6, 0, 0);
173  if (!m_bzfile) {
174  throw bzip2_error{"bzip2 error: write open failed", bzerror};
175  }
176  }
177 
178  Bzip2Compressor(const Bzip2Compressor&) = delete;
179  Bzip2Compressor& operator=(const Bzip2Compressor&) = delete;
180 
181  Bzip2Compressor(Bzip2Compressor&&) = delete;
182  Bzip2Compressor& operator=(Bzip2Compressor&&) = delete;
183 
184  ~Bzip2Compressor() noexcept final {
185  try {
186  close();
187  } catch (...) {
188  // Ignore any exceptions because destructor must not throw.
189  }
190  }
191 
192  void write(const std::string& data) final {
193  assert(data.size() < std::numeric_limits<int>::max());
194  assert(m_bzfile);
195 #ifdef _MSC_VER
196  osmium::detail::disable_invalid_parameter_handler diph;
197 #endif
198  int bzerror = BZ_OK;
199  ::BZ2_bzWrite(&bzerror, m_bzfile, const_cast<char*>(data.data()), static_cast<int>(data.size()));
200  if (bzerror != BZ_OK && bzerror != BZ_STREAM_END) {
201  detail::throw_bzip2_error(m_bzfile, "write failed", bzerror);
202  }
203  }
204 
205  void close() final {
206  if (m_bzfile) {
207 #ifdef _MSC_VER
208  osmium::detail::disable_invalid_parameter_handler diph;
209 #endif
210  int bzerror = BZ_OK;
211  ::BZ2_bzWriteClose(&bzerror, m_bzfile, 0, nullptr, nullptr);
212  m_bzfile = nullptr;
213  if (do_fsync() && m_file.file()) {
214  osmium::io::detail::reliable_fsync(fileno(m_file.file()));
215  }
216  m_file.close();
217  if (bzerror != BZ_OK) {
218  throw bzip2_error{"bzip2 error: write close failed", bzerror};
219  }
220  }
221  }
222 
223  }; // class Bzip2Compressor
224 
226 
227  detail::file_wrapper m_file;
228  BZFILE* m_bzfile = nullptr;
229  bool m_stream_end = false;
230 
231  public:
232 
233  explicit Bzip2Decompressor(const int fd) :
234  m_file(fd, "rb") {
235 #ifdef _MSC_VER
236  osmium::detail::disable_invalid_parameter_handler diph;
237 #endif
238  int bzerror = BZ_OK;
239  m_bzfile = ::BZ2_bzReadOpen(&bzerror, m_file.file(), 0, 0, nullptr, 0);
240  if (!m_bzfile) {
241  throw bzip2_error{"bzip2 error: read open failed", bzerror};
242  }
243  }
244 
245  Bzip2Decompressor(const Bzip2Decompressor&) = delete;
246  Bzip2Decompressor& operator=(const Bzip2Decompressor&) = delete;
247 
249  Bzip2Decompressor& operator=(Bzip2Decompressor&&) = delete;
250 
251  ~Bzip2Decompressor() noexcept final {
252  try {
253  close();
254  } catch (...) {
255  // Ignore any exceptions because destructor must not throw.
256  }
257  }
258 
259  std::string read() final {
260 #ifdef _MSC_VER
261  osmium::detail::disable_invalid_parameter_handler diph;
262 #endif
263  assert(m_bzfile);
264  std::string buffer;
265 
266  if (!m_stream_end) {
268  int bzerror = BZ_OK;
269  assert(buffer.size() < std::numeric_limits<int>::max());
270  const int nread = ::BZ2_bzRead(&bzerror, m_bzfile, &*buffer.begin(), static_cast<int>(buffer.size()));
271  if (bzerror != BZ_OK && bzerror != BZ_STREAM_END) {
272  detail::throw_bzip2_error(m_bzfile, "read failed", bzerror);
273  }
274  if (bzerror == BZ_STREAM_END) {
275  void* unused;
276  int nunused;
277  if (!feof(m_file.file())) {
278  ::BZ2_bzReadGetUnused(&bzerror, m_bzfile, &unused, &nunused);
279  if (bzerror != BZ_OK) {
280  detail::throw_bzip2_error(m_bzfile, "get unused failed", bzerror);
281  }
282  std::string unused_data{static_cast<const char*>(unused), static_cast<std::string::size_type>(nunused)};
283  ::BZ2_bzReadClose(&bzerror, m_bzfile);
284  if (bzerror != BZ_OK) {
285  throw bzip2_error{"bzip2 error: read close failed", bzerror};
286  }
287  assert(unused_data.size() < std::numeric_limits<int>::max());
288  m_bzfile = ::BZ2_bzReadOpen(&bzerror, m_file.file(), 0, 0, &*unused_data.begin(), static_cast<int>(unused_data.size()));
289  if (!m_bzfile) {
290  throw bzip2_error{"bzip2 error: read open failed", bzerror};
291  }
292  } else {
293  m_stream_end = true;
294  }
295  }
296  buffer.resize(static_cast<std::string::size_type>(nread));
297  }
298 
299  set_offset(static_cast<std::size_t>(ftell(m_file.file())));
300 
301  return buffer;
302  }
303 
304  void close() final {
305  if (m_bzfile) {
306 #ifdef _MSC_VER
307  osmium::detail::disable_invalid_parameter_handler diph;
308 #endif
309  int bzerror = BZ_OK;
310  ::BZ2_bzReadClose(&bzerror, m_bzfile);
311  m_bzfile = nullptr;
312  m_file.close();
313  if (bzerror != BZ_OK) {
314  throw bzip2_error{"bzip2 error: read close failed", bzerror};
315  }
316  }
317  }
318 
319  }; // class Bzip2Decompressor
320 
322 
323  const char* m_buffer;
324  std::size_t m_buffer_size;
325  bz_stream m_bzstream;
326 
327  public:
328 
329  Bzip2BufferDecompressor(const char* buffer, const std::size_t size) :
330  m_buffer(buffer),
331  m_buffer_size(size),
332  m_bzstream() {
333  m_bzstream.next_in = const_cast<char*>(buffer);
334  assert(size < std::numeric_limits<unsigned int>::max());
335  m_bzstream.avail_in = static_cast<unsigned int>(size);
336  const int result = BZ2_bzDecompressInit(&m_bzstream, 0, 0);
337  if (result != BZ_OK) {
338  throw bzip2_error{"bzip2 error: decompression init failed: ", result};
339  }
340  }
341 
343  Bzip2BufferDecompressor& operator=(const Bzip2BufferDecompressor&) = delete;
344 
346  Bzip2BufferDecompressor& operator=(Bzip2BufferDecompressor&&) = delete;
347 
348  ~Bzip2BufferDecompressor() noexcept final {
349  try {
350  close();
351  } catch (...) {
352  // Ignore any exceptions because destructor must not throw.
353  }
354  }
355 
356  std::string read() final {
357  std::string output;
358 
359  if (m_buffer) {
360  const std::size_t buffer_size = 10240;
361  output.resize(buffer_size);
362  m_bzstream.next_out = &*output.begin();
363  m_bzstream.avail_out = buffer_size;
364  const int result = BZ2_bzDecompress(&m_bzstream);
365 
366  if (result != BZ_OK) {
367  m_buffer = nullptr;
368  m_buffer_size = 0;
369  }
370 
371  if (result != BZ_OK && result != BZ_STREAM_END) {
372  throw bzip2_error{"bzip2 error: decompress failed: ", result};
373  }
374 
375  output.resize(static_cast<std::size_t>(m_bzstream.next_out - output.data()));
376  }
377 
378  return output;
379  }
380 
381  void close() final {
382  BZ2_bzDecompressEnd(&m_bzstream);
383  }
384 
385  }; // class Bzip2BufferDecompressor
386 
387  namespace detail {
388 
389  // we want the register_compression() function to run, setting
390  // the variable is only a side-effect, it will never be used
392  [](const int fd, const fsync sync) { return new osmium::io::Bzip2Compressor{fd, sync}; },
393  [](const int fd) { return new osmium::io::Bzip2Decompressor{fd}; },
394  [](const char* buffer, const std::size_t size) { return new osmium::io::Bzip2BufferDecompressor{buffer, size}; }
395  );
396 
397  // dummy function to silence the unused variable warning from above
398  inline bool get_registered_bzip2_compression() noexcept {
399  return registered_bzip2_compression;
400  }
401 
402  } // namespace detail
403 
404  } // namespace io
405 
406 } // namespace osmium
407 
408 #endif // OSMIUM_IO_BZIP2_COMPRESSION_HPP
bz_stream m_bzstream
Definition: bzip2_compression.hpp:325
Definition: bzip2_compression.hpp:225
Definition: compression.hpp:95
int bzip2_error_code
Definition: bzip2_compression.hpp:73
~Bzip2Decompressor() noexcept final
Definition: bzip2_compression.hpp:251
void close() final
Definition: bzip2_compression.hpp:381
void write(const std::string &data) final
Definition: bzip2_compression.hpp:192
const char * m_buffer
Definition: bzip2_compression.hpp:323
static CompressionFactory & instance()
Definition: compression.hpp:180
~Bzip2BufferDecompressor() noexcept final
Definition: bzip2_compression.hpp:348
bool register_compression(osmium::io::file_compression compression, const create_compressor_type &create_compressor, const create_decompressor_type_fd &create_decompressor_fd, const create_decompressor_type_buffer &create_decompressor_buffer)
Definition: compression.hpp:185
std::string read() final
Definition: bzip2_compression.hpp:259
Definition: bzip2_compression.hpp:158
int system_errno
Definition: bzip2_compression.hpp:74
Definition: compression.hpp:87
Bzip2Decompressor(const int fd)
Definition: bzip2_compression.hpp:233
Definition: bzip2_compression.hpp:321
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
Definition: attr.hpp:333
Bzip2BufferDecompressor(const char *buffer, const std::size_t size)
Definition: bzip2_compression.hpp:329
fsync
Definition: writer_options.hpp:51
Definition: error.hpp:44
void close() final
Definition: bzip2_compression.hpp:205
void close() final
Definition: bzip2_compression.hpp:304
detail::file_wrapper m_file
Definition: bzip2_compression.hpp:227
std::string read() final
Definition: bzip2_compression.hpp:356
Definition: compression.hpp:57
bzip2_error(const std::string &what, const int error_code)
Definition: bzip2_compression.hpp:76
Bzip2Compressor(const int fd, const fsync sync)
Definition: bzip2_compression.hpp:165
detail::file_wrapper m_file
Definition: bzip2_compression.hpp:160
std::size_t m_buffer_size
Definition: bzip2_compression.hpp:324
~Bzip2Compressor() noexcept final
Definition: bzip2_compression.hpp:184
Definition: bzip2_compression.hpp:71