Libosmium  2.17.0
Fast and flexible C++ library for working with OpenStreetMap data
compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_COMPRESSION_HPP
2 #define OSMIUM_IO_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2021 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <osmium/io/detail/read_write.hpp>
37 #include <osmium/io/error.hpp>
40 #include <osmium/util/file.hpp>
41 
42 #include <atomic>
43 #include <cerrno>
44 #include <cstddef>
45 #include <functional>
46 #include <map>
47 #include <memory>
48 #include <string>
49 #include <system_error>
50 #include <tuple>
51 #include <utility>
52 
53 namespace osmium {
54 
55  namespace io {
56 
57  class Compressor {
58 
60 
61  protected:
62 
63  bool do_fsync() const noexcept {
64  return m_fsync == fsync::yes;
65  }
66 
67  public:
68 
69  explicit Compressor(const fsync sync) noexcept :
70  m_fsync(sync) {
71  }
72 
73  Compressor(const Compressor&) = default;
74  Compressor& operator=(const Compressor&) = default;
75 
76  Compressor(Compressor&&) noexcept = default;
77  Compressor& operator=(Compressor&&) noexcept = default;
78 
79  virtual ~Compressor() noexcept = default;
80 
81  virtual void write(const std::string& data) = 0;
82 
83  virtual void close() = 0;
84 
85  virtual std::size_t file_size() const {
86  return 0;
87  }
88 
89  }; // class Compressor
90 
91  class Decompressor {
92 
93  std::atomic<std::size_t> m_file_size{0};
94  std::atomic<std::size_t> m_offset{0};
95 
96  public:
97 
98  enum {
99  input_buffer_size = 1024U * 1024U
100  };
101 
102  Decompressor() = default;
103 
104  Decompressor(const Decompressor&) = delete;
106 
109 
110  virtual ~Decompressor() noexcept = default;
111 
112  virtual std::string read() = 0;
113 
114  virtual void close() = 0;
115 
116  std::size_t file_size() const noexcept {
117  return m_file_size;
118  }
119 
120  void set_file_size(const std::size_t size) noexcept {
121  m_file_size = size;
122  }
123 
124  std::size_t offset() const noexcept {
125  return m_offset;
126  }
127 
128  void set_offset(const std::size_t offset) noexcept {
129  m_offset = offset;
130  }
131 
132  }; // class Decompressor
133 
142 
143  public:
144 
145  using create_compressor_type = std::function<osmium::io::Compressor*(int, fsync)>;
147  using create_decompressor_type_buffer = std::function<osmium::io::Decompressor*(const char*, std::size_t)>;
148 
149  private:
150 
154 
155  using compression_map_type = std::map<const osmium::io::file_compression, callbacks_type>;
156 
158 
159  CompressionFactory() = default;
160 
162  const auto it = m_callbacks.find(compression);
163 
164  if (it != m_callbacks.end()) {
165  return it->second;
166  }
167 
168  std::string error_message{"Support for compression '"};
169  error_message += as_string(compression);
170  error_message += "' not compiled into this binary";
171  throw unsupported_file_format_error{error_message};
172  }
173 
174  public:
175 
178 
181 
182  ~CompressionFactory() noexcept = default;
183 
185  static CompressionFactory factory;
186  return factory;
187  }
188 
190  osmium::io::file_compression compression,
192  const create_decompressor_type_fd& create_decompressor_fd,
193  const create_decompressor_type_buffer& create_decompressor_buffer) {
194 
195  compression_map_type::value_type cc{compression,
196  std::make_tuple(create_compressor,
197  create_decompressor_fd,
198  create_decompressor_buffer)};
199 
200  return m_callbacks.insert(cc).second;
201  }
202 
203  template <typename... TArgs>
204  std::unique_ptr<osmium::io::Compressor> create_compressor(const osmium::io::file_compression compression, TArgs&&... args) const {
205  const auto callbacks = find_callbacks(compression);
206  return std::unique_ptr<osmium::io::Compressor>(std::get<0>(callbacks)(std::forward<TArgs>(args)...));
207  }
208 
209  std::unique_ptr<osmium::io::Decompressor> create_decompressor(const osmium::io::file_compression compression, const int fd) const {
210  const auto callbacks = find_callbacks(compression);
211  auto p = std::unique_ptr<osmium::io::Decompressor>(std::get<1>(callbacks)(fd));
212  p->set_file_size(osmium::file_size(fd));
213  return p;
214  }
215 
216  std::unique_ptr<osmium::io::Decompressor> create_decompressor(const osmium::io::file_compression compression, const char* buffer, const std::size_t size) const {
217  const auto callbacks = find_callbacks(compression);
218  return std::unique_ptr<osmium::io::Decompressor>(std::get<2>(callbacks)(buffer, size));
219  }
220 
221  }; // class CompressionFactory
222 
223  class NoCompressor final : public Compressor {
224 
225  std::size_t m_file_size = 0;
226  int m_fd;
227 
228  public:
229 
230  NoCompressor(const int fd, const fsync sync) :
231  Compressor(sync),
232  m_fd(fd) {
233  }
234 
235  NoCompressor(const NoCompressor&) = delete;
237 
240 
241  ~NoCompressor() noexcept override {
242  try {
243  close();
244  } catch (...) {
245  // Ignore any exceptions because destructor must not throw.
246  }
247  }
248 
249  void write(const std::string& data) override {
250  osmium::io::detail::reliable_write(m_fd, data.data(), data.size());
251  m_file_size += data.size();
252  }
253 
254  void close() override {
255  if (m_fd >= 0) {
256  const int fd = m_fd;
257  m_fd = -1;
258 
259  // Do not sync or close stdout
260  if (fd == 1) {
261  return;
262  }
263 
264  if (do_fsync()) {
265  osmium::io::detail::reliable_fsync(fd);
266  }
267  osmium::io::detail::reliable_close(fd);
268  }
269  }
270 
271  std::size_t file_size() const override {
272  return m_file_size;
273  }
274 
275  }; // class NoCompressor
276 
277  class NoDecompressor final : public Decompressor {
278 
279  int m_fd = -1;
280  const char* m_buffer = nullptr;
281  std::size_t m_buffer_size = 0;
282  std::size_t m_offset = 0;
283 
284  public:
285 
286  explicit NoDecompressor(const int fd) :
287  m_fd(fd) {
288  }
289 
290  NoDecompressor(const char* buffer, const std::size_t size) :
291  m_buffer(buffer),
292  m_buffer_size(size) {
293  }
294 
295  NoDecompressor(const NoDecompressor&) = delete;
297 
300 
301  ~NoDecompressor() noexcept override {
302  try {
303  close();
304  } catch (...) {
305  // Ignore any exceptions because destructor must not throw.
306  }
307  }
308 
309  std::string read() override {
310  std::string buffer;
311 
312  if (m_buffer) {
313  if (m_buffer_size != 0) {
314  const std::size_t size = m_buffer_size;
315  m_buffer_size = 0;
316  buffer.append(m_buffer, size);
317  }
318  } else {
320  osmium::io::detail::remove_buffered_pages(m_fd, m_offset);
321  const auto nread = detail::reliable_read(m_fd, &*buffer.begin(), osmium::io::Decompressor::input_buffer_size);
322  buffer.resize(std::string::size_type(nread));
323  }
324 
325  m_offset += buffer.size();
327 
328  return buffer;
329  }
330 
331  void close() override {
332  if (m_fd >= 0) {
333  osmium::io::detail::remove_buffered_pages(m_fd);
334  const int fd = m_fd;
335  m_fd = -1;
336  osmium::io::detail::reliable_close(fd);
337  }
338  }
339 
340  }; // class NoDecompressor
341 
342  namespace detail {
343 
344  // we want the register_compression() function to run, setting
345  // the variable is only a side-effect, it will never be used
347  [](const int fd, const fsync sync) { return new osmium::io::NoCompressor{fd, sync}; },
348  [](const int fd) { return new osmium::io::NoDecompressor{fd}; },
349  [](const char* buffer, std::size_t size) { return new osmium::io::NoDecompressor{buffer, size}; }
350  );
351 
352  // dummy function to silence the unused variable warning from above
353  inline bool get_registered_no_compression() noexcept {
354  return registered_no_compression;
355  }
356 
357  } // namespace detail
358 
359  } // namespace io
360 
361 } // namespace osmium
362 
363 #endif // OSMIUM_IO_COMPRESSION_HPP
Definition: compression.hpp:141
~CompressionFactory() noexcept=default
bool register_compression(osmium::io::file_compression compression, const create_compressor_type &create_compressor, const create_decompressor_type_fd &create_decompressor_fd, const create_decompressor_type_buffer &create_decompressor_buffer)
Definition: compression.hpp:189
compression_map_type m_callbacks
Definition: compression.hpp:157
std::unique_ptr< osmium::io::Decompressor > create_decompressor(const osmium::io::file_compression compression, const int fd) const
Definition: compression.hpp:209
std::function< osmium::io::Compressor *(int, fsync)> create_compressor_type
Definition: compression.hpp:145
CompressionFactory & operator=(CompressionFactory &&)=delete
CompressionFactory & operator=(const CompressionFactory &)=delete
std::map< const osmium::io::file_compression, callbacks_type > compression_map_type
Definition: compression.hpp:155
std::function< osmium::io::Decompressor *(int)> create_decompressor_type_fd
Definition: compression.hpp:146
std::unique_ptr< osmium::io::Decompressor > create_decompressor(const osmium::io::file_compression compression, const char *buffer, const std::size_t size) const
Definition: compression.hpp:216
std::unique_ptr< osmium::io::Compressor > create_compressor(const osmium::io::file_compression compression, TArgs &&... args) const
Definition: compression.hpp:204
CompressionFactory(const CompressionFactory &)=delete
static CompressionFactory & instance()
Definition: compression.hpp:184
std::function< osmium::io::Decompressor *(const char *, std::size_t)> create_decompressor_type_buffer
Definition: compression.hpp:147
CompressionFactory(CompressionFactory &&)=delete
const callbacks_type & find_callbacks(const osmium::io::file_compression compression) const
Definition: compression.hpp:161
std::tuple< create_compressor_type, create_decompressor_type_fd, create_decompressor_type_buffer > callbacks_type
Definition: compression.hpp:153
Definition: compression.hpp:57
bool do_fsync() const noexcept
Definition: compression.hpp:63
Compressor(const fsync sync) noexcept
Definition: compression.hpp:69
Compressor(const Compressor &)=default
Compressor(Compressor &&) noexcept=default
fsync m_fsync
Definition: compression.hpp:59
virtual void write(const std::string &data)=0
virtual void close()=0
Compressor & operator=(const Compressor &)=default
virtual std::size_t file_size() const
Definition: compression.hpp:85
Definition: compression.hpp:91
@ input_buffer_size
Definition: compression.hpp:99
Decompressor & operator=(const Decompressor &)=delete
Decompressor & operator=(Decompressor &&)=delete
std::size_t offset() const noexcept
Definition: compression.hpp:124
virtual std::string read()=0
void set_file_size(const std::size_t size) noexcept
Definition: compression.hpp:120
std::size_t file_size() const noexcept
Definition: compression.hpp:116
virtual void close()=0
std::atomic< std::size_t > m_file_size
Definition: compression.hpp:93
virtual ~Decompressor() noexcept=default
void set_offset(const std::size_t offset) noexcept
Definition: compression.hpp:128
std::atomic< std::size_t > m_offset
Definition: compression.hpp:94
Decompressor(const Decompressor &)=delete
Decompressor(Decompressor &&)=delete
Definition: compression.hpp:223
void close() override
Definition: compression.hpp:254
void write(const std::string &data) override
Definition: compression.hpp:249
NoCompressor(const NoCompressor &)=delete
~NoCompressor() noexcept override
Definition: compression.hpp:241
std::size_t m_file_size
Definition: compression.hpp:225
NoCompressor & operator=(const NoCompressor &)=delete
NoCompressor & operator=(NoCompressor &&)=delete
std::size_t file_size() const override
Definition: compression.hpp:271
NoCompressor(const int fd, const fsync sync)
Definition: compression.hpp:230
NoCompressor(NoCompressor &&)=delete
int m_fd
Definition: compression.hpp:226
Definition: compression.hpp:277
const char * m_buffer
Definition: compression.hpp:280
NoDecompressor(const NoDecompressor &)=delete
int m_fd
Definition: compression.hpp:279
NoDecompressor(const char *buffer, const std::size_t size)
Definition: compression.hpp:290
std::size_t m_buffer_size
Definition: compression.hpp:281
std::size_t m_offset
Definition: compression.hpp:282
NoDecompressor & operator=(NoDecompressor &&)=delete
NoDecompressor & operator=(const NoDecompressor &)=delete
std::string read() override
Definition: compression.hpp:309
NoDecompressor(NoDecompressor &&)=delete
~NoDecompressor() noexcept override
Definition: compression.hpp:301
NoDecompressor(const int fd)
Definition: compression.hpp:286
void close() override
Definition: compression.hpp:331
Definition: attr.hpp:342
file_compression
Definition: file_compression.hpp:42
const char * as_string(file_compression compression)
Definition: file_compression.hpp:48
fsync
Definition: writer_options.hpp:51
std::size_t file_size(int fd)
Definition: file.hpp:109
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
Definition: location.hpp:551