Libosmium  2.15.6
Fast and flexible C++ library for working with OpenStreetMap data
compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_COMPRESSION_HPP
2 #define OSMIUM_IO_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2020 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <osmium/io/detail/read_write.hpp>
37 #include <osmium/io/error.hpp>
40 #include <osmium/util/file.hpp>
41 
42 #include <atomic>
43 #include <cerrno>
44 #include <cstddef>
45 #include <functional>
46 #include <map>
47 #include <memory>
48 #include <string>
49 #include <system_error>
50 #include <tuple>
51 #include <utility>
52 
53 namespace osmium {
54 
55  namespace io {
56 
57  class Compressor {
58 
60 
61  protected:
62 
63  bool do_fsync() const noexcept {
64  return m_fsync == fsync::yes;
65  }
66 
67  public:
68 
69  explicit Compressor(const fsync sync) noexcept :
70  m_fsync(sync) {
71  }
72 
73  Compressor(const Compressor&) = default;
74  Compressor& operator=(const Compressor&) = default;
75 
76  Compressor(Compressor&&) noexcept = default;
77  Compressor& operator=(Compressor&&) noexcept = default;
78 
79  virtual ~Compressor() noexcept = default;
80 
81  virtual void write(const std::string& data) = 0;
82 
83  virtual void close() = 0;
84 
85  }; // class Compressor
86 
87  class Decompressor {
88 
89  std::atomic<std::size_t> m_file_size{0};
90  std::atomic<std::size_t> m_offset{0};
91 
92  public:
93 
94  enum {
95  input_buffer_size = 1024U * 1024U
96  };
97 
98  Decompressor() = default;
99 
100  Decompressor(const Decompressor&) = delete;
101  Decompressor& operator=(const Decompressor&) = delete;
102 
103  Decompressor(Decompressor&&) = delete;
104  Decompressor& operator=(Decompressor&&) = delete;
105 
106  virtual ~Decompressor() noexcept = default;
107 
108  virtual std::string read() = 0;
109 
110  virtual void close() = 0;
111 
112  std::size_t file_size() const noexcept {
113  return m_file_size;
114  }
115 
116  void set_file_size(const std::size_t size) noexcept {
117  m_file_size = size;
118  }
119 
120  std::size_t offset() const noexcept {
121  return m_offset;
122  }
123 
124  void set_offset(const std::size_t offset) noexcept {
125  m_offset = offset;
126  }
127 
128  }; // class Decompressor
129 
138 
139  public:
140 
141  using create_compressor_type = std::function<osmium::io::Compressor*(int, fsync)>;
142  using create_decompressor_type_fd = std::function<osmium::io::Decompressor*(int)>;
143  using create_decompressor_type_buffer = std::function<osmium::io::Decompressor*(const char*, std::size_t)>;
144 
145  private:
146 
147  using callbacks_type = std::tuple<create_compressor_type,
150 
151  using compression_map_type = std::map<const osmium::io::file_compression, callbacks_type>;
152 
154 
155  CompressionFactory() = default;
156 
158  const auto it = m_callbacks.find(compression);
159 
160  if (it != m_callbacks.end()) {
161  return it->second;
162  }
163 
164  std::string error_message{"Support for compression '"};
165  error_message += as_string(compression);
166  error_message += "' not compiled into this binary";
167  throw unsupported_file_format_error{error_message};
168  }
169 
170  public:
171 
172  CompressionFactory(const CompressionFactory&) = delete;
174 
177 
178  ~CompressionFactory() noexcept = default;
179 
181  static CompressionFactory factory;
182  return factory;
183  }
184 
186  osmium::io::file_compression compression,
187  const create_compressor_type& create_compressor,
188  const create_decompressor_type_fd& create_decompressor_fd,
189  const create_decompressor_type_buffer& create_decompressor_buffer) {
190 
191  compression_map_type::value_type cc{compression,
192  std::make_tuple(create_compressor,
193  create_decompressor_fd,
194  create_decompressor_buffer)};
195 
196  return m_callbacks.insert(cc).second;
197  }
198 
199  template <typename... TArgs>
200  std::unique_ptr<osmium::io::Compressor> create_compressor(const osmium::io::file_compression compression, TArgs&&... args) const {
201  const auto callbacks = find_callbacks(compression);
202  return std::unique_ptr<osmium::io::Compressor>(std::get<0>(callbacks)(std::forward<TArgs>(args)...));
203  }
204 
205  std::unique_ptr<osmium::io::Decompressor> create_decompressor(const osmium::io::file_compression compression, const int fd) const {
206  const auto callbacks = find_callbacks(compression);
207  auto p = std::unique_ptr<osmium::io::Decompressor>(std::get<1>(callbacks)(fd));
208  p->set_file_size(osmium::file_size(fd));
209  return p;
210  }
211 
212  std::unique_ptr<osmium::io::Decompressor> create_decompressor(const osmium::io::file_compression compression, const char* buffer, const std::size_t size) const {
213  const auto callbacks = find_callbacks(compression);
214  return std::unique_ptr<osmium::io::Decompressor>(std::get<2>(callbacks)(buffer, size));
215  }
216 
217  }; // class CompressionFactory
218 
219  class NoCompressor final : public Compressor {
220 
221  int m_fd;
222 
223  public:
224 
225  NoCompressor(const int fd, const fsync sync) :
226  Compressor(sync),
227  m_fd(fd) {
228  }
229 
230  NoCompressor(const NoCompressor&) = delete;
231  NoCompressor& operator=(const NoCompressor&) = delete;
232 
233  NoCompressor(NoCompressor&&) = delete;
234  NoCompressor& operator=(NoCompressor&&) = delete;
235 
236  ~NoCompressor() noexcept {
237  try {
238  close();
239  } catch (...) {
240  // Ignore any exceptions because destructor must not throw.
241  }
242  }
243 
244  void write(const std::string& data) override {
245  osmium::io::detail::reliable_write(m_fd, data.data(), data.size());
246  }
247 
248  void close() override {
249  if (m_fd >= 0) {
250  const int fd = m_fd;
251  m_fd = -1;
252 
253  // Do not sync or close stdout
254  if (fd == 1) {
255  return;
256  }
257 
258  if (do_fsync()) {
259  osmium::io::detail::reliable_fsync(fd);
260  }
261  osmium::io::detail::reliable_close(fd);
262  }
263  }
264 
265  }; // class NoCompressor
266 
267  class NoDecompressor final : public Decompressor {
268 
269  int m_fd = -1;
270  const char* m_buffer = nullptr;
271  std::size_t m_buffer_size = 0;
272  std::size_t m_offset = 0;
273 
274  public:
275 
276  explicit NoDecompressor(const int fd) :
277  m_fd(fd) {
278  }
279 
280  NoDecompressor(const char* buffer, const std::size_t size) :
281  m_buffer(buffer),
282  m_buffer_size(size) {
283  }
284 
285  NoDecompressor(const NoDecompressor&) = delete;
286  NoDecompressor& operator=(const NoDecompressor&) = delete;
287 
288  NoDecompressor(NoDecompressor&&) = delete;
290 
291  ~NoDecompressor() noexcept {
292  try {
293  close();
294  } catch (...) {
295  // Ignore any exceptions because destructor must not throw.
296  }
297  }
298 
299  std::string read() override {
300  std::string buffer;
301 
302  if (m_buffer) {
303  if (m_buffer_size != 0) {
304  const std::size_t size = m_buffer_size;
305  m_buffer_size = 0;
306  buffer.append(m_buffer, size);
307  }
308  } else {
310  const auto nread = detail::reliable_read(m_fd, &*buffer.begin(), osmium::io::Decompressor::input_buffer_size);
311  buffer.resize(std::string::size_type(nread));
312  }
313 
314  m_offset += buffer.size();
315  set_offset(m_offset);
316 
317  return buffer;
318  }
319 
320  void close() override {
321  if (m_fd >= 0) {
322  const int fd = m_fd;
323  m_fd = -1;
324  osmium::io::detail::reliable_close(fd);
325  }
326  }
327 
328  }; // class NoDecompressor
329 
330  namespace detail {
331 
332  // we want the register_compression() function to run, setting
333  // the variable is only a side-effect, it will never be used
335  [](const int fd, const fsync sync) { return new osmium::io::NoCompressor{fd, sync}; },
336  [](const int fd) { return new osmium::io::NoDecompressor{fd}; },
337  [](const char* buffer, std::size_t size) { return new osmium::io::NoDecompressor{buffer, size}; }
338  );
339 
340  // dummy function to silence the unused variable warning from above
341  inline bool get_registered_no_compression() noexcept {
342  return registered_no_compression;
343  }
344 
345  } // namespace detail
346 
347  } // namespace io
348 
349 } // namespace osmium
350 
351 #endif // OSMIUM_IO_COMPRESSION_HPP
Definition: compression.hpp:95
NoDecompressor(const char *buffer, const std::size_t size)
Definition: compression.hpp:280
std::size_t file_size(int fd)
Definition: file.hpp:109
std::unique_ptr< osmium::io::Compressor > create_compressor(const osmium::io::file_compression compression, TArgs &&... args) const
Definition: compression.hpp:200
static CompressionFactory & instance()
Definition: compression.hpp:180
void write(const std::string &data) override
Definition: compression.hpp:244
Compressor(const fsync sync) noexcept
Definition: compression.hpp:69
Definition: location.hpp:551
std::tuple< create_compressor_type, create_decompressor_type_fd, create_decompressor_type_buffer > callbacks_type
Definition: compression.hpp:149
void close() override
Definition: compression.hpp:320
~NoDecompressor() noexcept
Definition: compression.hpp:291
bool register_compression(osmium::io::file_compression compression, const create_compressor_type &create_compressor, const create_decompressor_type_fd &create_decompressor_fd, const create_decompressor_type_buffer &create_decompressor_buffer)
Definition: compression.hpp:185
virtual ~Compressor() noexcept=default
Definition: compression.hpp:87
fsync m_fsync
Definition: compression.hpp:59
void set_offset(const std::size_t offset) noexcept
Definition: compression.hpp:124
void set_file_size(const std::size_t size) noexcept
Definition: compression.hpp:116
bool do_fsync() const noexcept
Definition: compression.hpp:63
void close() override
Definition: compression.hpp:248
virtual void write(const std::string &data)=0
std::unique_ptr< osmium::io::Decompressor > create_decompressor(const osmium::io::file_compression compression, const char *buffer, const std::size_t size) const
Definition: compression.hpp:212
const char * as_string(file_compression compression)
Definition: file_compression.hpp:48
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
Definition: attr.hpp:342
const callbacks_type & find_callbacks(const osmium::io::file_compression compression) const
Definition: compression.hpp:157
fsync
Definition: writer_options.hpp:51
virtual void close()=0
std::size_t offset() const noexcept
Definition: compression.hpp:120
std::function< osmium::io::Decompressor *(const char *, std::size_t)> create_decompressor_type_buffer
Definition: compression.hpp:143
std::map< const osmium::io::file_compression, callbacks_type > compression_map_type
Definition: compression.hpp:151
file_compression
Definition: file_compression.hpp:42
std::function< osmium::io::Compressor *(int, fsync)> create_compressor_type
Definition: compression.hpp:141
Compressor & operator=(const Compressor &)=default
Definition: compression.hpp:219
Definition: compression.hpp:57
~NoCompressor() noexcept
Definition: compression.hpp:236
int m_fd
Definition: compression.hpp:221
NoDecompressor(const int fd)
Definition: compression.hpp:276
std::string read() override
Definition: compression.hpp:299
compression_map_type m_callbacks
Definition: compression.hpp:153
std::function< osmium::io::Decompressor *(int)> create_decompressor_type_fd
Definition: compression.hpp:142
Definition: compression.hpp:267
std::unique_ptr< osmium::io::Decompressor > create_decompressor(const osmium::io::file_compression compression, const int fd) const
Definition: compression.hpp:205
Definition: compression.hpp:137
NoCompressor(const int fd, const fsync sync)
Definition: compression.hpp:225