Libosmium  2.9.0
Fast and flexible C++ library for working with OpenStreetMap data
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_COMPRESSION_HPP
2 #define OSMIUM_IO_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <atomic>
37 #include <cerrno>
38 #include <cstddef>
39 #include <functional>
40 #include <map>
41 #include <memory>
42 #include <string>
43 #include <system_error>
44 #include <tuple>
45 #include <utility>
46 
47 #ifndef _MSC_VER
48 # include <unistd.h>
49 #else
50 # include <io.h>
51 #endif
52 
53 #include <osmium/io/detail/read_write.hpp>
54 #include <osmium/io/error.hpp>
58 #include <osmium/util/file.hpp>
59 
60 namespace osmium {
61 
62  namespace io {
63 
64  class Compressor {
65 
67 
68  protected:
69 
70  bool do_fsync() const {
71  return m_fsync == fsync::yes;
72  }
73 
74  public:
75 
76  explicit Compressor(fsync sync) :
77  m_fsync(sync) {
78  }
79 
80  virtual ~Compressor() noexcept {
81  }
82 
83  virtual void write(const std::string& data) = 0;
84 
85  virtual void close() = 0;
86 
87  }; // class Compressor
88 
89  class Decompressor {
90 
91  std::atomic<size_t> m_file_size {0};
92  std::atomic<size_t> m_offset {0};
93 
94  public:
95 
96  static constexpr unsigned int input_buffer_size = 1024 * 1024;
97 
98  Decompressor() = default;
99 
100  Decompressor(const Decompressor&) = delete;
101  Decompressor& operator=(const Decompressor&) = delete;
102 
103  Decompressor(Decompressor&&) = delete;
104  Decompressor& operator=(Decompressor&&) = delete;
105 
106  virtual ~Decompressor() noexcept {
107  }
108 
109  virtual std::string read() = 0;
110 
111  virtual void close() = 0;
112 
113  size_t file_size() const noexcept {
114  return m_file_size;
115  }
116 
117  void set_file_size(size_t size) noexcept {
118  m_file_size = size;
119  }
120 
121  size_t offset() const noexcept {
122  return m_offset;
123  }
124 
125  void set_offset(size_t offset) noexcept {
126  m_offset = offset;
127  }
128 
129  }; // class Decompressor
130 
139 
140  public:
141 
142  using create_compressor_type = std::function<osmium::io::Compressor*(int, fsync)>;
143  using create_decompressor_type_fd = std::function<osmium::io::Decompressor*(int)>;
144  using create_decompressor_type_buffer = std::function<osmium::io::Decompressor*(const char*, size_t)>;
145 
146  private:
147 
148  using compression_map_type = std::map<const osmium::io::file_compression,
149  std::tuple<create_compressor_type,
152 
154 
155  CompressionFactory() = default;
156 
157  CompressionFactory(const CompressionFactory&) = delete;
159 
162 
163  OSMIUM_NORETURN void error(osmium::io::file_compression compression) {
164  std::string error_message {"Support for compression '"};
165  error_message += as_string(compression);
166  error_message += "' not compiled into this binary.";
167  throw unsupported_file_format_error(error_message);
168  }
169 
170  public:
171 
173  static CompressionFactory factory;
174  return factory;
175  }
176 
178  osmium::io::file_compression compression,
179  create_compressor_type create_compressor,
180  create_decompressor_type_fd create_decompressor_fd,
181  create_decompressor_type_buffer create_decompressor_buffer) {
182 
183  compression_map_type::value_type cc(compression,
184  std::make_tuple(create_compressor,
185  create_decompressor_fd,
186  create_decompressor_buffer));
187 
188  return m_callbacks.insert(cc).second;
189  }
190 
191  template <typename... TArgs>
192  std::unique_ptr<osmium::io::Compressor> create_compressor(osmium::io::file_compression compression, TArgs&&... args) {
193  auto it = m_callbacks.find(compression);
194 
195  if (it != m_callbacks.end()) {
196  return std::unique_ptr<osmium::io::Compressor>(std::get<0>(it->second)(std::forward<TArgs>(args)...));
197  }
198 
199  error(compression);
200  }
201 
202  std::unique_ptr<osmium::io::Decompressor> create_decompressor(osmium::io::file_compression compression, int fd) {
203  auto it = m_callbacks.find(compression);
204 
205  if (it != m_callbacks.end()) {
206  auto p = std::unique_ptr<osmium::io::Decompressor>(std::get<1>(it->second)(fd));
207  p->set_file_size(osmium::util::file_size(fd));
208  return p;
209  }
210 
211  error(compression);
212  }
213 
214  std::unique_ptr<osmium::io::Decompressor> create_decompressor(osmium::io::file_compression compression, const char* buffer, size_t size) {
215  auto it = m_callbacks.find(compression);
216 
217  if (it != m_callbacks.end()) {
218  return std::unique_ptr<osmium::io::Decompressor>(std::get<2>(it->second)(buffer, size));
219  }
220 
221  error(compression);
222  }
223 
224  }; // class CompressionFactory
225 
226  class NoCompressor : public Compressor {
227 
228  int m_fd;
229 
230  public:
231 
232  NoCompressor(int fd, fsync sync) :
233  Compressor(sync),
234  m_fd(fd) {
235  }
236 
237  ~NoCompressor() noexcept final {
238  try {
239  close();
240  } catch (...) {
241  // Ignore any exceptions because destructor must not throw.
242  }
243  }
244 
245  void write(const std::string& data) final {
246  osmium::io::detail::reliable_write(m_fd, data.data(), data.size());
247  }
248 
249  void close() final {
250  if (m_fd >= 0) {
251  int fd = m_fd;
252  m_fd = -1;
253  if (do_fsync()) {
254  osmium::io::detail::reliable_fsync(fd);
255  }
256  osmium::io::detail::reliable_close(fd);
257  }
258  }
259 
260  }; // class NoCompressor
261 
262  class NoDecompressor : public Decompressor {
263 
264  int m_fd;
265  const char *m_buffer;
267  size_t m_offset = 0;
268 
269  public:
270 
271  explicit NoDecompressor(int fd) :
272  Decompressor(),
273  m_fd(fd),
274  m_buffer(nullptr),
275  m_buffer_size(0) {
276  }
277 
278  NoDecompressor(const char* buffer, size_t size) :
279  Decompressor(),
280  m_fd(-1),
281  m_buffer(buffer),
282  m_buffer_size(size) {
283  }
284 
285  ~NoDecompressor() noexcept final {
286  try {
287  close();
288  } catch (...) {
289  // Ignore any exceptions because destructor must not throw.
290  }
291  }
292 
293  std::string read() final {
294  std::string buffer;
295 
296  if (m_buffer) {
297  if (m_buffer_size != 0) {
298  size_t size = m_buffer_size;
299  m_buffer_size = 0;
300  buffer.append(m_buffer, size);
301  }
302  } else {
304  auto nread = ::read(m_fd, const_cast<char*>(buffer.data()), osmium::io::Decompressor::input_buffer_size);
305  if (nread < 0) {
306  throw std::system_error(errno, std::system_category(), "Read failed");
307  }
308  buffer.resize(std::string::size_type(nread));
309  }
310 
311  m_offset += buffer.size();
312  set_offset(m_offset);
313 
314  return buffer;
315  }
316 
317  void close() final {
318  if (m_fd >= 0) {
319  int fd = m_fd;
320  m_fd = -1;
321  osmium::io::detail::reliable_close(fd);
322  }
323  }
324 
325  }; // class NoDecompressor
326 
327  namespace detail {
328 
329  // we want the register_compression() function to run, setting
330  // the variable is only a side-effect, it will never be used
332  [](int fd, fsync sync) { return new osmium::io::NoCompressor(fd, sync); },
333  [](int fd) { return new osmium::io::NoDecompressor(fd); },
334  [](const char* buffer, size_t size) { return new osmium::io::NoDecompressor(buffer, size); }
335  );
336 
337  // dummy function to silence the unused variable warning from above
338  inline bool get_registered_no_compression() noexcept {
339  return registered_no_compression;
340  }
341 
342  } // namespace detail
343 
344  } // namespace io
345 
346 } // namespace osmium
347 
348 #endif // OSMIUM_IO_COMPRESSION_HPP
OSMIUM_NORETURN void error(osmium::io::file_compression compression)
Definition: compression.hpp:163
std::map< const osmium::io::file_compression, std::tuple< create_compressor_type, create_decompressor_type_fd, create_decompressor_type_buffer >> compression_map_type
Definition: compression.hpp:151
void close() final
Definition: compression.hpp:317
#define OSMIUM_NORETURN
Definition: compatibility.hpp:41
NoDecompressor(int fd)
Definition: compression.hpp:271
virtual std::string read()=0
Decompressor & operator=(const Decompressor &)=delete
size_t file_size(int fd)
Definition: file.hpp:67
std::string read() final
Definition: compression.hpp:293
static CompressionFactory & instance()
Definition: compression.hpp:172
static constexpr unsigned int input_buffer_size
Definition: compression.hpp:96
void close() final
Definition: compression.hpp:249
void write(const std::string &data) final
Definition: compression.hpp:245
~NoCompressor() noexceptfinal
Definition: compression.hpp:237
Definition: compression.hpp:89
fsync m_fsync
Definition: compression.hpp:66
virtual void write(const std::string &data)=0
Compressor(fsync sync)
Definition: compression.hpp:76
int m_fd
Definition: compression.hpp:264
const char * as_string(file_compression compression)
Definition: file_compression.hpp:51
size_t m_buffer_size
Definition: compression.hpp:266
Namespace for everything in the Osmium library.
Definition: assembler.hpp:73
Definition: attr.hpp:333
~NoDecompressor() noexceptfinal
Definition: compression.hpp:285
std::unique_ptr< osmium::io::Decompressor > create_decompressor(osmium::io::file_compression compression, int fd)
Definition: compression.hpp:202
size_t offset() const noexcept
Definition: compression.hpp:121
fsync
Definition: writer_options.hpp:51
std::unique_ptr< osmium::io::Decompressor > create_decompressor(osmium::io::file_compression compression, const char *buffer, size_t size)
Definition: compression.hpp:214
virtual void close()=0
size_t file_size() const noexcept
Definition: compression.hpp:113
void set_file_size(size_t size) noexcept
Definition: compression.hpp:117
size_t m_offset
Definition: compression.hpp:267
bool do_fsync() const
Definition: compression.hpp:70
file_compression
Definition: file_compression.hpp:42
std::function< osmium::io::Compressor *(int, fsync)> create_compressor_type
Definition: compression.hpp:142
std::unique_ptr< osmium::io::Compressor > create_compressor(osmium::io::file_compression compression, TArgs &&...args)
Definition: compression.hpp:192
std::atomic< size_t > m_file_size
Definition: compression.hpp:91
virtual ~Decompressor() noexcept
Definition: compression.hpp:106
Definition: compression.hpp:226
Definition: compression.hpp:64
int m_fd
Definition: compression.hpp:228
compression_map_type m_callbacks
Definition: compression.hpp:153
CompressionFactory & operator=(const CompressionFactory &)=delete
std::function< osmium::io::Decompressor *(int)> create_decompressor_type_fd
Definition: compression.hpp:143
std::function< osmium::io::Decompressor *(const char *, size_t)> create_decompressor_type_buffer
Definition: compression.hpp:144
virtual ~Compressor() noexcept
Definition: compression.hpp:80
Definition: compression.hpp:262
Definition: compression.hpp:138
NoDecompressor(const char *buffer, size_t size)
Definition: compression.hpp:278
bool register_compression(osmium::io::file_compression compression, create_compressor_type create_compressor, create_decompressor_type_fd create_decompressor_fd, create_decompressor_type_buffer create_decompressor_buffer)
Definition: compression.hpp:177
const char * m_buffer
Definition: compression.hpp:265
NoCompressor(int fd, fsync sync)
Definition: compression.hpp:232
std::atomic< size_t > m_offset
Definition: compression.hpp:92
void set_offset(size_t offset) noexcept
Definition: compression.hpp:125
virtual void close()=0