Libosmium  2.14.0
Fast and flexible C++ library for working with OpenStreetMap data
compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_COMPRESSION_HPP
2 #define OSMIUM_IO_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2018 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <osmium/io/detail/read_write.hpp>
37 #include <osmium/io/error.hpp>
40 #include <osmium/util/file.hpp>
41 
42 #include <atomic>
43 #include <cerrno>
44 #include <cstddef>
45 #include <functional>
46 #include <map>
47 #include <memory>
48 #include <string>
49 #include <system_error>
50 #include <tuple>
51 #include <utility>
52 
53 #ifndef _MSC_VER
54 # include <unistd.h>
55 #else
56 # include <io.h>
57 #endif
58 
59 namespace osmium {
60 
61  namespace io {
62 
63  class Compressor {
64 
66 
67  protected:
68 
69  bool do_fsync() const {
70  return m_fsync == fsync::yes;
71  }
72 
73  public:
74 
75  explicit Compressor(fsync sync) :
76  m_fsync(sync) {
77  }
78 
79  Compressor(const Compressor&) = default;
80  Compressor& operator=(const Compressor&) = default;
81 
82  Compressor(Compressor&&) noexcept = default;
83  Compressor& operator=(Compressor&&) noexcept = default;
84 
85  virtual ~Compressor() noexcept = default;
86 
87  virtual void write(const std::string& data) = 0;
88 
89  virtual void close() = 0;
90 
91  }; // class Compressor
92 
93  class Decompressor {
94 
95  std::atomic<std::size_t> m_file_size{0};
96  std::atomic<std::size_t> m_offset{0};
97 
98  public:
99 
100  static constexpr unsigned int input_buffer_size = 1024 * 1024;
101 
102  Decompressor() = default;
103 
104  Decompressor(const Decompressor&) = delete;
105  Decompressor& operator=(const Decompressor&) = delete;
106 
107  Decompressor(Decompressor&&) = delete;
108  Decompressor& operator=(Decompressor&&) = delete;
109 
110  virtual ~Decompressor() noexcept = default;
111 
112  virtual std::string read() = 0;
113 
114  virtual void close() = 0;
115 
116  std::size_t file_size() const noexcept {
117  return m_file_size;
118  }
119 
120  void set_file_size(std::size_t size) noexcept {
121  m_file_size = size;
122  }
123 
124  std::size_t offset() const noexcept {
125  return m_offset;
126  }
127 
128  void set_offset(std::size_t offset) noexcept {
129  m_offset = offset;
130  }
131 
132  }; // class Decompressor
133 
142 
143  public:
144 
145  using create_compressor_type = std::function<osmium::io::Compressor*(int, fsync)>;
146  using create_decompressor_type_fd = std::function<osmium::io::Decompressor*(int)>;
147  using create_decompressor_type_buffer = std::function<osmium::io::Decompressor*(const char*, std::size_t)>;
148 
149  private:
150 
151  using callbacks_type = std::tuple<create_compressor_type,
154 
155  using compression_map_type = std::map<const osmium::io::file_compression, callbacks_type>;
156 
158 
159  CompressionFactory() = default;
160 
162  const auto it = m_callbacks.find(compression);
163 
164  if (it != m_callbacks.end()) {
165  return it->second;
166  }
167 
168  std::string error_message{"Support for compression '"};
169  error_message += as_string(compression);
170  error_message += "' not compiled into this binary";
171  throw unsupported_file_format_error{error_message};
172  }
173 
174  public:
175 
176  CompressionFactory(const CompressionFactory&) = delete;
178 
181 
182  ~CompressionFactory() noexcept = default;
183 
185  static CompressionFactory factory;
186  return factory;
187  }
188 
190  osmium::io::file_compression compression,
191  create_compressor_type create_compressor,
192  create_decompressor_type_fd create_decompressor_fd,
193  create_decompressor_type_buffer create_decompressor_buffer) {
194 
195  compression_map_type::value_type cc{compression,
196  std::make_tuple(create_compressor,
197  create_decompressor_fd,
198  create_decompressor_buffer)};
199 
200  return m_callbacks.insert(cc).second;
201  }
202 
203  template <typename... TArgs>
204  std::unique_ptr<osmium::io::Compressor> create_compressor(osmium::io::file_compression compression, TArgs&&... args) const {
205  const auto callbacks = find_callbacks(compression);
206  return std::unique_ptr<osmium::io::Compressor>(std::get<0>(callbacks)(std::forward<TArgs>(args)...));
207  }
208 
209  std::unique_ptr<osmium::io::Decompressor> create_decompressor(osmium::io::file_compression compression, int fd) const {
210  const auto callbacks = find_callbacks(compression);
211  auto p = std::unique_ptr<osmium::io::Decompressor>(std::get<1>(callbacks)(fd));
212  p->set_file_size(osmium::file_size(fd));
213  return p;
214  }
215 
216  std::unique_ptr<osmium::io::Decompressor> create_decompressor(osmium::io::file_compression compression, const char* buffer, std::size_t size) const {
217  const auto callbacks = find_callbacks(compression);
218  return std::unique_ptr<osmium::io::Decompressor>(std::get<2>(callbacks)(buffer, size));
219  }
220 
221  }; // class CompressionFactory
222 
223  class NoCompressor : public Compressor {
224 
225  int m_fd;
226 
227  public:
228 
229  NoCompressor(int fd, fsync sync) :
230  Compressor(sync),
231  m_fd(fd) {
232  }
233 
234  NoCompressor(const NoCompressor&) = delete;
235  NoCompressor& operator=(const NoCompressor&) = delete;
236 
237  NoCompressor(NoCompressor&&) = delete;
238  NoCompressor& operator=(NoCompressor&&) = delete;
239 
240  ~NoCompressor() noexcept final {
241  try {
242  close();
243  } catch (...) {
244  // Ignore any exceptions because destructor must not throw.
245  }
246  }
247 
248  void write(const std::string& data) final {
249  osmium::io::detail::reliable_write(m_fd, data.data(), data.size());
250  }
251 
252  void close() final {
253  if (m_fd >= 0) {
254  const int fd = m_fd;
255  m_fd = -1;
256  if (do_fsync()) {
257  osmium::io::detail::reliable_fsync(fd);
258  }
259  osmium::io::detail::reliable_close(fd);
260  }
261  }
262 
263  }; // class NoCompressor
264 
265  class NoDecompressor : public Decompressor {
266 
267  int m_fd = -1;
268  const char* m_buffer = nullptr;
269  std::size_t m_buffer_size = 0;
270  std::size_t m_offset = 0;
271 
272  public:
273 
274  explicit NoDecompressor(int fd) :
275  m_fd(fd) {
276  }
277 
278  NoDecompressor(const char* buffer, std::size_t size) :
279  m_buffer(buffer),
280  m_buffer_size(size) {
281  }
282 
283  NoDecompressor(const NoDecompressor&) = delete;
284  NoDecompressor& operator=(const NoDecompressor&) = delete;
285 
286  NoDecompressor(NoDecompressor&&) = delete;
288 
289  ~NoDecompressor() noexcept final {
290  try {
291  close();
292  } catch (...) {
293  // Ignore any exceptions because destructor must not throw.
294  }
295  }
296 
297  std::string read() final {
298  std::string buffer;
299 
300  if (m_buffer) {
301  if (m_buffer_size != 0) {
302  const std::size_t size = m_buffer_size;
303  m_buffer_size = 0;
304  buffer.append(m_buffer, size);
305  }
306  } else {
308  const auto nread = detail::reliable_read(m_fd, const_cast<char*>(buffer.data()), osmium::io::Decompressor::input_buffer_size);
309  buffer.resize(std::string::size_type(nread));
310  }
311 
312  m_offset += buffer.size();
313  set_offset(m_offset);
314 
315  return buffer;
316  }
317 
318  void close() final {
319  if (m_fd >= 0) {
320  const int fd = m_fd;
321  m_fd = -1;
322  osmium::io::detail::reliable_close(fd);
323  }
324  }
325 
326  }; // class NoDecompressor
327 
328  namespace detail {
329 
330  // we want the register_compression() function to run, setting
331  // the variable is only a side-effect, it will never be used
333  [](int fd, fsync sync) { return new osmium::io::NoCompressor{fd, sync}; },
334  [](int fd) { return new osmium::io::NoDecompressor{fd}; },
335  [](const char* buffer, std::size_t size) { return new osmium::io::NoDecompressor{buffer, size}; }
336  );
337 
338  // dummy function to silence the unused variable warning from above
339  inline bool get_registered_no_compression() noexcept {
340  return registered_no_compression;
341  }
342 
343  } // namespace detail
344 
345  } // namespace io
346 
347 } // namespace osmium
348 
349 #endif // OSMIUM_IO_COMPRESSION_HPP
void set_offset(std::size_t offset) noexcept
Definition: compression.hpp:128
void close() final
Definition: compression.hpp:318
NoDecompressor(int fd)
Definition: compression.hpp:274
std::size_t file_size(int fd)
Definition: file.hpp:109
std::string read() final
Definition: compression.hpp:297
const callbacks_type & find_callbacks(osmium::io::file_compression compression) const
Definition: compression.hpp:161
static CompressionFactory & instance()
Definition: compression.hpp:184
static constexpr unsigned int input_buffer_size
Definition: compression.hpp:100
void close() final
Definition: compression.hpp:252
Definition: location.hpp:550
void write(const std::string &data) final
Definition: compression.hpp:248
std::tuple< create_compressor_type, create_decompressor_type_fd, create_decompressor_type_buffer > callbacks_type
Definition: compression.hpp:153
virtual ~Compressor() noexcept=default
Definition: compression.hpp:93
fsync m_fsync
Definition: compression.hpp:65
void set_file_size(std::size_t size) noexcept
Definition: compression.hpp:120
virtual void write(const std::string &data)=0
Compressor(fsync sync)
Definition: compression.hpp:75
NoDecompressor(const char *buffer, std::size_t size)
Definition: compression.hpp:278
const char * as_string(file_compression compression)
Definition: file_compression.hpp:48
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
Definition: attr.hpp:333
fsync
Definition: writer_options.hpp:51
bool do_fsync() const
Definition: compression.hpp:69
virtual void close()=0
std::size_t offset() const noexcept
Definition: compression.hpp:124
~NoCompressor() noexcept final
Definition: compression.hpp:240
std::function< osmium::io::Decompressor *(const char *, std::size_t)> create_decompressor_type_buffer
Definition: compression.hpp:147
std::map< const osmium::io::file_compression, callbacks_type > compression_map_type
Definition: compression.hpp:155
file_compression
Definition: file_compression.hpp:42
std::function< osmium::io::Compressor *(int, fsync)> create_compressor_type
Definition: compression.hpp:145
Compressor & operator=(const Compressor &)=default
Definition: compression.hpp:223
std::unique_ptr< osmium::io::Decompressor > create_decompressor(osmium::io::file_compression compression, int fd) const
Definition: compression.hpp:209
Definition: compression.hpp:63
int m_fd
Definition: compression.hpp:225
compression_map_type m_callbacks
Definition: compression.hpp:157
std::function< osmium::io::Decompressor *(int)> create_decompressor_type_fd
Definition: compression.hpp:146
Definition: compression.hpp:265
Definition: compression.hpp:141
std::unique_ptr< osmium::io::Compressor > create_compressor(osmium::io::file_compression compression, TArgs &&... args) const
Definition: compression.hpp:204
std::unique_ptr< osmium::io::Decompressor > create_decompressor(osmium::io::file_compression compression, const char *buffer, std::size_t size) const
Definition: compression.hpp:216
bool register_compression(osmium::io::file_compression compression, create_compressor_type create_compressor, create_decompressor_type_fd create_decompressor_fd, create_decompressor_type_buffer create_decompressor_buffer)
Definition: compression.hpp:189
NoCompressor(int fd, fsync sync)
Definition: compression.hpp:229
~NoDecompressor() noexcept final
Definition: compression.hpp:289