Libosmium  2.14.0
Fast and flexible C++ library for working with OpenStreetMap data
gzip_compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_GZIP_COMPRESSION_HPP
2 #define OSMIUM_IO_GZIP_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2018 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
46 #include <osmium/io/detail/read_write.hpp>
47 #include <osmium/io/error.hpp>
51 
52 #include <zlib.h>
53 
54 #include <cassert>
55 #include <cerrno>
56 #include <cstddef>
57 #include <limits>
58 #include <string>
59 
60 #ifndef _MSC_VER
61 # include <unistd.h>
62 #endif
63 
64 namespace osmium {
65 
70  struct gzip_error : public io_error {
71 
74 
75  gzip_error(const std::string& what, int error_code) :
76  io_error(what),
77  gzip_error_code(error_code),
78  system_errno(error_code == Z_ERRNO ? errno : 0) {
79  }
80 
81  }; // struct gzip_error
82 
83  namespace io {
84 
85  namespace detail {
86 
87  OSMIUM_NORETURN inline void throw_gzip_error(gzFile gzfile, const char* msg, int zlib_error = 0) {
88  std::string error{"gzip error: "};
89  error += msg;
90  error += ": ";
91  int errnum = zlib_error;
92  if (zlib_error) {
93  error += std::to_string(zlib_error);
94  } else {
95  error += ::gzerror(gzfile, &errnum);
96  }
97  throw osmium::gzip_error{error, errnum};
98  }
99 
100  } // namespace detail
101 
102  class GzipCompressor : public Compressor {
103 
104  int m_fd;
105  gzFile m_gzfile;
106 
107  public:
108 
109  explicit GzipCompressor(int fd, fsync sync) :
110  Compressor(sync),
111  m_fd(::dup(fd)),
112  m_gzfile(::gzdopen(fd, "w")) {
113  if (!m_gzfile) {
114  detail::throw_gzip_error(m_gzfile, "write initialization failed");
115  }
116  }
117 
118  GzipCompressor(const GzipCompressor&) = delete;
119  GzipCompressor& operator=(const GzipCompressor&) = delete;
120 
121  GzipCompressor(GzipCompressor&&) = delete;
122  GzipCompressor& operator=(GzipCompressor&&) = delete;
123 
124  ~GzipCompressor() noexcept final {
125  try {
126  close();
127  } catch (...) {
128  // Ignore any exceptions because destructor must not throw.
129  }
130  }
131 
132  void write(const std::string& data) final {
133  if (!data.empty()) {
134  assert(data.size() < std::numeric_limits<unsigned int>::max());
135  const int nwrite = ::gzwrite(m_gzfile, data.data(), static_cast<unsigned int>(data.size()));
136  if (nwrite == 0) {
137  detail::throw_gzip_error(m_gzfile, "write failed");
138  }
139  }
140  }
141 
142  void close() final {
143  if (m_gzfile) {
144  const int result = ::gzclose(m_gzfile);
145  m_gzfile = nullptr;
146  if (result != Z_OK) {
147  detail::throw_gzip_error(m_gzfile, "write close failed", result);
148  }
149  if (do_fsync()) {
150  osmium::io::detail::reliable_fsync(m_fd);
151  }
152  osmium::io::detail::reliable_close(m_fd);
153  }
154  }
155 
156  }; // class GzipCompressor
157 
159 
160  gzFile m_gzfile;
161 
162  public:
163 
164  explicit GzipDecompressor(int fd) :
165  m_gzfile(::gzdopen(fd, "r")) {
166  if (!m_gzfile) {
167  detail::throw_gzip_error(m_gzfile, "read initialization failed");
168  }
169  }
170 
171  GzipDecompressor(const GzipDecompressor&) = delete;
172  GzipDecompressor& operator=(const GzipDecompressor&) = delete;
173 
175  GzipDecompressor& operator=(GzipDecompressor&&) = delete;
176 
177  ~GzipDecompressor() noexcept final {
178  try {
179  close();
180  } catch (...) {
181  // Ignore any exceptions because destructor must not throw.
182  }
183  }
184 
185  std::string read() final {
186  std::string buffer(osmium::io::Decompressor::input_buffer_size, '\0');
187  assert(buffer.size() < std::numeric_limits<unsigned int>::max());
188  int nread = ::gzread(m_gzfile, const_cast<char*>(buffer.data()), static_cast<unsigned int>(buffer.size()));
189  if (nread < 0) {
190  detail::throw_gzip_error(m_gzfile, "read failed");
191  }
192  buffer.resize(static_cast<std::string::size_type>(nread));
193 #if ZLIB_VERNUM >= 0x1240
194  set_offset(size_t(::gzoffset(m_gzfile)));
195 #endif
196  return buffer;
197  }
198 
199  void close() final {
200  if (m_gzfile) {
201  const int result = ::gzclose(m_gzfile);
202  m_gzfile = nullptr;
203  if (result != Z_OK) {
204  detail::throw_gzip_error(m_gzfile, "read close failed", result);
205  }
206  }
207  }
208 
209  }; // class GzipDecompressor
210 
212 
213  const char* m_buffer;
215  z_stream m_zstream;
216 
217  public:
218 
219  GzipBufferDecompressor(const char* buffer, size_t size) :
220  m_buffer(buffer),
221  m_buffer_size(size),
222  m_zstream() {
223  m_zstream.next_in = reinterpret_cast<unsigned char*>(const_cast<char*>(buffer));
224  assert(size < std::numeric_limits<unsigned int>::max());
225  m_zstream.avail_in = static_cast<unsigned int>(size);
226  const int result = inflateInit2(&m_zstream, MAX_WBITS | 32); // NOLINT(hicpp-signed-bitwise)
227  if (result != Z_OK) {
228  std::string message{"gzip error: decompression init failed: "};
229  if (m_zstream.msg) {
230  message.append(m_zstream.msg);
231  }
232  throw osmium::gzip_error{message, result};
233  }
234  }
235 
237  GzipBufferDecompressor& operator=(const GzipBufferDecompressor&) = delete;
238 
240  GzipBufferDecompressor& operator=(GzipBufferDecompressor&&) = delete;
241 
242  ~GzipBufferDecompressor() noexcept final {
243  try {
244  close();
245  } catch (...) {
246  // Ignore any exceptions because destructor must not throw.
247  }
248  }
249 
250  std::string read() final {
251  std::string output;
252 
253  if (m_buffer) {
254  const size_t buffer_size = 10240;
255  output.append(buffer_size, '\0');
256  m_zstream.next_out = reinterpret_cast<unsigned char*>(const_cast<char*>(output.data()));
257  m_zstream.avail_out = buffer_size;
258  const int result = inflate(&m_zstream, Z_SYNC_FLUSH);
259 
260  if (result != Z_OK) {
261  m_buffer = nullptr;
262  m_buffer_size = 0;
263  }
264 
265  if (result != Z_OK && result != Z_STREAM_END) {
266  std::string message("gzip error: inflate failed: ");
267  if (m_zstream.msg) {
268  message.append(m_zstream.msg);
269  }
270  throw osmium::gzip_error{message, result};
271  }
272 
273  output.resize(static_cast<std::size_t>(m_zstream.next_out - reinterpret_cast<const unsigned char*>(output.data())));
274  }
275 
276  return output;
277  }
278 
279  void close() final {
280  inflateEnd(&m_zstream);
281  }
282 
283  }; // class GzipBufferDecompressor
284 
285  namespace detail {
286 
287  // we want the register_compression() function to run, setting
288  // the variable is only a side-effect, it will never be used
290  [](int fd, fsync sync) { return new osmium::io::GzipCompressor{fd, sync}; },
291  [](int fd) { return new osmium::io::GzipDecompressor{fd}; },
292  [](const char* buffer, size_t size) { return new osmium::io::GzipBufferDecompressor{buffer, size}; }
293  );
294 
295  // dummy function to silence the unused variable warning from above
296  inline bool get_registered_gzip_compression() noexcept {
297  return registered_gzip_compression;
298  }
299 
300  } // namespace detail
301 
302  } // namespace io
303 
304 } // namespace osmium
305 
306 #endif // OSMIUM_IO_GZIP_COMPRESSION_HPP
gzFile m_gzfile
Definition: gzip_compression.hpp:105
Definition: gzip_compression.hpp:211
gzip_error(const std::string &what, int error_code)
Definition: gzip_compression.hpp:75
gzFile m_gzfile
Definition: gzip_compression.hpp:160
#define OSMIUM_NORETURN
Definition: compatibility.hpp:41
Definition: gzip_compression.hpp:102
z_stream m_zstream
Definition: gzip_compression.hpp:215
static CompressionFactory & instance()
Definition: compression.hpp:184
static constexpr unsigned int input_buffer_size
Definition: compression.hpp:100
GzipCompressor(int fd, fsync sync)
Definition: gzip_compression.hpp:109
size_t m_buffer_size
Definition: gzip_compression.hpp:214
Definition: gzip_compression.hpp:158
~GzipDecompressor() noexcept final
Definition: gzip_compression.hpp:177
Definition: compression.hpp:93
int m_fd
Definition: gzip_compression.hpp:104
GzipBufferDecompressor(const char *buffer, size_t size)
Definition: gzip_compression.hpp:219
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
const char * m_buffer
Definition: gzip_compression.hpp:213
Definition: attr.hpp:333
void close() final
Definition: gzip_compression.hpp:279
fsync
Definition: writer_options.hpp:51
void close() final
Definition: gzip_compression.hpp:142
Definition: error.hpp:44
int system_errno
Definition: gzip_compression.hpp:73
Definition: gzip_compression.hpp:70
std::string read() final
Definition: gzip_compression.hpp:185
int gzip_error_code
Definition: gzip_compression.hpp:72
GzipDecompressor(int fd)
Definition: gzip_compression.hpp:164
Definition: compression.hpp:63
void write(const std::string &data) final
Definition: gzip_compression.hpp:132
~GzipCompressor() noexcept final
Definition: gzip_compression.hpp:124
~GzipBufferDecompressor() noexcept final
Definition: gzip_compression.hpp:242
bool register_compression(osmium::io::file_compression compression, create_compressor_type create_compressor, create_decompressor_type_fd create_decompressor_fd, create_decompressor_type_buffer create_decompressor_buffer)
Definition: compression.hpp:189
std::string read() final
Definition: gzip_compression.hpp:250
void close() final
Definition: gzip_compression.hpp:199