Libosmium  2.13.1
Fast and flexible C++ library for working with OpenStreetMap data
compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_COMPRESSION_HPP
2 #define OSMIUM_IO_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2017 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <atomic>
37 #include <cerrno>
38 #include <cstddef>
39 #include <functional>
40 #include <map>
41 #include <memory>
42 #include <string>
43 #include <system_error>
44 #include <tuple>
45 #include <utility>
46 
47 #ifndef _MSC_VER
48 # include <unistd.h>
49 #else
50 # include <io.h>
51 #endif
52 
53 #include <osmium/io/detail/read_write.hpp>
54 #include <osmium/io/error.hpp>
57 #include <osmium/util/file.hpp>
58 
59 namespace osmium {
60 
61  namespace io {
62 
63  class Compressor {
64 
66 
67  protected:
68 
69  bool do_fsync() const {
70  return m_fsync == fsync::yes;
71  }
72 
73  public:
74 
75  explicit Compressor(fsync sync) :
76  m_fsync(sync) {
77  }
78 
79  virtual ~Compressor() noexcept = default;
80 
81  virtual void write(const std::string& data) = 0;
82 
83  virtual void close() = 0;
84 
85  }; // class Compressor
86 
87  class Decompressor {
88 
89  std::atomic<std::size_t> m_file_size{0};
90  std::atomic<std::size_t> m_offset{0};
91 
92  public:
93 
94  static constexpr unsigned int input_buffer_size = 1024 * 1024;
95 
96  Decompressor() = default;
97 
98  Decompressor(const Decompressor&) = delete;
99  Decompressor& operator=(const Decompressor&) = delete;
100 
101  Decompressor(Decompressor&&) = delete;
102  Decompressor& operator=(Decompressor&&) = delete;
103 
104  virtual ~Decompressor() noexcept = default;
105 
106  virtual std::string read() = 0;
107 
108  virtual void close() = 0;
109 
110  std::size_t file_size() const noexcept {
111  return m_file_size;
112  }
113 
114  void set_file_size(std::size_t size) noexcept {
115  m_file_size = size;
116  }
117 
118  std::size_t offset() const noexcept {
119  return m_offset;
120  }
121 
122  void set_offset(std::size_t offset) noexcept {
123  m_offset = offset;
124  }
125 
126  }; // class Decompressor
127 
136 
137  public:
138 
139  using create_compressor_type = std::function<osmium::io::Compressor*(int, fsync)>;
140  using create_decompressor_type_fd = std::function<osmium::io::Decompressor*(int)>;
141  using create_decompressor_type_buffer = std::function<osmium::io::Decompressor*(const char*, std::size_t)>;
142 
143  private:
144 
145  using callbacks_type = std::tuple<create_compressor_type,
148 
149  using compression_map_type = std::map<const osmium::io::file_compression, callbacks_type>;
150 
152 
153  CompressionFactory() = default;
154 
155  CompressionFactory(const CompressionFactory&) = delete;
156  CompressionFactory& operator=(const CompressionFactory&) = delete;
157 
159  CompressionFactory& operator=(CompressionFactory&&) = delete;
160 
162  const auto it = m_callbacks.find(compression);
163 
164  if (it != m_callbacks.end()) {
165  return it->second;
166  }
167 
168  std::string error_message{"Support for compression '"};
169  error_message += as_string(compression);
170  error_message += "' not compiled into this binary";
171  throw unsupported_file_format_error{error_message};
172  }
173 
174  public:
175 
177  static CompressionFactory factory;
178  return factory;
179  }
180 
182  osmium::io::file_compression compression,
183  create_compressor_type create_compressor,
184  create_decompressor_type_fd create_decompressor_fd,
185  create_decompressor_type_buffer create_decompressor_buffer) {
186 
187  compression_map_type::value_type cc{compression,
188  std::make_tuple(create_compressor,
189  create_decompressor_fd,
190  create_decompressor_buffer)};
191 
192  return m_callbacks.insert(cc).second;
193  }
194 
195  template <typename... TArgs>
196  std::unique_ptr<osmium::io::Compressor> create_compressor(osmium::io::file_compression compression, TArgs&&... args) const {
197  const auto callbacks = find_callbacks(compression);
198  return std::unique_ptr<osmium::io::Compressor>(std::get<0>(callbacks)(std::forward<TArgs>(args)...));
199  }
200 
201  std::unique_ptr<osmium::io::Decompressor> create_decompressor(osmium::io::file_compression compression, int fd) const {
202  const auto callbacks = find_callbacks(compression);
203  auto p = std::unique_ptr<osmium::io::Decompressor>(std::get<1>(callbacks)(fd));
204  p->set_file_size(osmium::util::file_size(fd));
205  return p;
206  }
207 
208  std::unique_ptr<osmium::io::Decompressor> create_decompressor(osmium::io::file_compression compression, const char* buffer, std::size_t size) const {
209  const auto callbacks = find_callbacks(compression);
210  return std::unique_ptr<osmium::io::Decompressor>(std::get<2>(callbacks)(buffer, size));
211  }
212 
213  }; // class CompressionFactory
214 
215  class NoCompressor : public Compressor {
216 
217  int m_fd;
218 
219  public:
220 
221  NoCompressor(int fd, fsync sync) :
222  Compressor(sync),
223  m_fd(fd) {
224  }
225 
226  ~NoCompressor() noexcept final {
227  try {
228  close();
229  } catch (...) {
230  // Ignore any exceptions because destructor must not throw.
231  }
232  }
233 
234  void write(const std::string& data) final {
235  osmium::io::detail::reliable_write(m_fd, data.data(), data.size());
236  }
237 
238  void close() final {
239  if (m_fd >= 0) {
240  const int fd = m_fd;
241  m_fd = -1;
242  if (do_fsync()) {
243  osmium::io::detail::reliable_fsync(fd);
244  }
245  osmium::io::detail::reliable_close(fd);
246  }
247  }
248 
249  }; // class NoCompressor
250 
251  class NoDecompressor : public Decompressor {
252 
253  int m_fd;
254  const char *m_buffer;
255  std::size_t m_buffer_size;
256  std::size_t m_offset = 0;
257 
258  public:
259 
260  explicit NoDecompressor(int fd) :
261  Decompressor(),
262  m_fd(fd),
263  m_buffer(nullptr),
264  m_buffer_size(0) {
265  }
266 
267  NoDecompressor(const char* buffer, std::size_t size) :
268  Decompressor(),
269  m_fd(-1),
270  m_buffer(buffer),
271  m_buffer_size(size) {
272  }
273 
274  ~NoDecompressor() noexcept final {
275  try {
276  close();
277  } catch (...) {
278  // Ignore any exceptions because destructor must not throw.
279  }
280  }
281 
282  std::string read() final {
283  std::string buffer;
284 
285  if (m_buffer) {
286  if (m_buffer_size != 0) {
287  const std::size_t size = m_buffer_size;
288  m_buffer_size = 0;
289  buffer.append(m_buffer, size);
290  }
291  } else {
293  const auto nread = ::read(m_fd, const_cast<char*>(buffer.data()), osmium::io::Decompressor::input_buffer_size);
294  if (nread < 0) {
295  throw std::system_error{errno, std::system_category(), "Read failed"};
296  }
297  buffer.resize(std::string::size_type(nread));
298  }
299 
300  m_offset += buffer.size();
301  set_offset(m_offset);
302 
303  return buffer;
304  }
305 
306  void close() final {
307  if (m_fd >= 0) {
308  const int fd = m_fd;
309  m_fd = -1;
310  osmium::io::detail::reliable_close(fd);
311  }
312  }
313 
314  }; // class NoDecompressor
315 
316  namespace detail {
317 
318  // we want the register_compression() function to run, setting
319  // the variable is only a side-effect, it will never be used
321  [](int fd, fsync sync) { return new osmium::io::NoCompressor{fd, sync}; },
322  [](int fd) { return new osmium::io::NoDecompressor{fd}; },
323  [](const char* buffer, std::size_t size) { return new osmium::io::NoDecompressor{buffer, size}; }
324  );
325 
326  // dummy function to silence the unused variable warning from above
327  inline bool get_registered_no_compression() noexcept {
328  return registered_no_compression;
329  }
330 
331  } // namespace detail
332 
333  } // namespace io
334 
335 } // namespace osmium
336 
337 #endif // OSMIUM_IO_COMPRESSION_HPP
void set_offset(std::size_t offset) noexcept
Definition: compression.hpp:122
void close() final
Definition: compression.hpp:306
NoDecompressor(int fd)
Definition: compression.hpp:260
std::size_t file_size(int fd)
Definition: file.hpp:109
std::string read() final
Definition: compression.hpp:282
const callbacks_type & find_callbacks(osmium::io::file_compression compression) const
Definition: compression.hpp:161
static CompressionFactory & instance()
Definition: compression.hpp:176
static constexpr unsigned int input_buffer_size
Definition: compression.hpp:94
void close() final
Definition: compression.hpp:238
Definition: reader_iterator.hpp:39
void write(const std::string &data) final
Definition: compression.hpp:234
std::tuple< create_compressor_type, create_decompressor_type_fd, create_decompressor_type_buffer > callbacks_type
Definition: compression.hpp:147
virtual ~Compressor() noexcept=default
Definition: compression.hpp:87
fsync m_fsync
Definition: compression.hpp:65
void set_file_size(std::size_t size) noexcept
Definition: compression.hpp:114
std::size_t m_buffer_size
Definition: compression.hpp:255
virtual void write(const std::string &data)=0
Compressor(fsync sync)
Definition: compression.hpp:75
int m_fd
Definition: compression.hpp:253
NoDecompressor(const char *buffer, std::size_t size)
Definition: compression.hpp:267
const char * as_string(file_compression compression)
Definition: file_compression.hpp:48
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
Definition: attr.hpp:333
fsync
Definition: writer_options.hpp:51
bool do_fsync() const
Definition: compression.hpp:69
virtual void close()=0
std::size_t offset() const noexcept
Definition: compression.hpp:118
~NoCompressor() noexcept final
Definition: compression.hpp:226
std::function< osmium::io::Decompressor *(const char *, std::size_t)> create_decompressor_type_buffer
Definition: compression.hpp:141
std::map< const osmium::io::file_compression, callbacks_type > compression_map_type
Definition: compression.hpp:149
file_compression
Definition: file_compression.hpp:42
std::function< osmium::io::Compressor *(int, fsync)> create_compressor_type
Definition: compression.hpp:139
Definition: compression.hpp:215
std::unique_ptr< osmium::io::Decompressor > create_decompressor(osmium::io::file_compression compression, int fd) const
Definition: compression.hpp:201
Definition: compression.hpp:63
int m_fd
Definition: compression.hpp:217
compression_map_type m_callbacks
Definition: compression.hpp:151
std::function< osmium::io::Decompressor *(int)> create_decompressor_type_fd
Definition: compression.hpp:140
Definition: compression.hpp:251
Definition: compression.hpp:135
std::unique_ptr< osmium::io::Compressor > create_compressor(osmium::io::file_compression compression, TArgs &&... args) const
Definition: compression.hpp:196
std::unique_ptr< osmium::io::Decompressor > create_decompressor(osmium::io::file_compression compression, const char *buffer, std::size_t size) const
Definition: compression.hpp:208
bool register_compression(osmium::io::file_compression compression, create_compressor_type create_compressor, create_decompressor_type_fd create_decompressor_fd, create_decompressor_type_buffer create_decompressor_buffer)
Definition: compression.hpp:181
const char * m_buffer
Definition: compression.hpp:254
NoCompressor(int fd, fsync sync)
Definition: compression.hpp:221
~NoDecompressor() noexcept final
Definition: compression.hpp:274