Libosmium  2.6.0
Fast and flexible C++ library for working with OpenStreetMap data
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_COMPRESSION_HPP
2 #define OSMIUM_IO_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <cerrno>
37 #include <functional>
38 #include <map>
39 #include <memory>
40 #include <stdexcept>
41 #include <string>
42 #include <system_error>
43 #include <tuple>
44 #include <utility>
45 
46 #ifndef _MSC_VER
47 # include <unistd.h>
48 #else
49 # include <io.h>
50 #endif
51 
52 #include <osmium/io/detail/read_write.hpp>
53 #include <osmium/io/error.hpp>
57 
58 namespace osmium {
59 
60  namespace io {
61 
62  class Compressor {
63 
65 
66  protected:
67 
68  bool do_fsync() const {
69  return m_fsync == fsync::yes;
70  }
71 
72  public:
73 
74  explicit Compressor(fsync sync) :
75  m_fsync(sync) {
76  }
77 
78  virtual ~Compressor() noexcept {
79  }
80 
81  virtual void write(const std::string& data) = 0;
82 
83  virtual void close() = 0;
84 
85  }; // class Compressor
86 
87  class Decompressor {
88 
89  public:
90 
91  static constexpr unsigned int input_buffer_size = 1024 * 1024;
92 
93  Decompressor() = default;
94 
95  Decompressor(const Decompressor&) = delete;
96  Decompressor& operator=(const Decompressor&) = delete;
97 
98  Decompressor(Decompressor&&) = delete;
99  Decompressor& operator=(Decompressor&&) = delete;
100 
101  virtual ~Decompressor() noexcept {
102  }
103 
104  virtual std::string read() = 0;
105 
106  virtual void close() = 0;
107 
108  }; // class Decompressor
109 
118 
119  public:
120 
121  typedef std::function<osmium::io::Compressor*(int, fsync)> create_compressor_type;
122  typedef std::function<osmium::io::Decompressor*(int)> create_decompressor_type_fd;
123  typedef std::function<osmium::io::Decompressor*(const char*, size_t)> create_decompressor_type_buffer;
124 
125  private:
126 
127  typedef std::map<const osmium::io::file_compression,
128  std::tuple<create_compressor_type,
130  create_decompressor_type_buffer>> compression_map_type;
131 
132  compression_map_type m_callbacks;
133 
134  CompressionFactory() = default;
135 
136  CompressionFactory(const CompressionFactory&) = delete;
138 
141 
142  OSMIUM_NORETURN void error(osmium::io::file_compression compression) {
143  std::string error_message {"Support for compression '"};
144  error_message += as_string(compression);
145  error_message += "' not compiled into this binary.";
146  throw unsupported_file_format_error(error_message);
147  }
148 
149  public:
150 
152  static CompressionFactory factory;
153  return factory;
154  }
155 
157  osmium::io::file_compression compression,
158  create_compressor_type create_compressor,
159  create_decompressor_type_fd create_decompressor_fd,
160  create_decompressor_type_buffer create_decompressor_buffer) {
161 
162  compression_map_type::value_type cc(compression,
163  std::make_tuple(create_compressor,
164  create_decompressor_fd,
165  create_decompressor_buffer));
166 
167  return m_callbacks.insert(cc).second;
168  }
169 
170  template <typename... TArgs>
171  std::unique_ptr<osmium::io::Compressor> create_compressor(osmium::io::file_compression compression, TArgs&&... args) {
172  auto it = m_callbacks.find(compression);
173 
174  if (it != m_callbacks.end()) {
175  return std::unique_ptr<osmium::io::Compressor>(std::get<0>(it->second)(std::forward<TArgs>(args)...));
176  }
177 
178  error(compression);
179  }
180 
181  std::unique_ptr<osmium::io::Decompressor> create_decompressor(osmium::io::file_compression compression, int fd) {
182  auto it = m_callbacks.find(compression);
183 
184  if (it != m_callbacks.end()) {
185  return std::unique_ptr<osmium::io::Decompressor>(std::get<1>(it->second)(fd));
186  }
187 
188  error(compression);
189  }
190 
191  std::unique_ptr<osmium::io::Decompressor> create_decompressor(osmium::io::file_compression compression, const char* buffer, size_t size) {
192  auto it = m_callbacks.find(compression);
193 
194  if (it != m_callbacks.end()) {
195  return std::unique_ptr<osmium::io::Decompressor>(std::get<2>(it->second)(buffer, size));
196  }
197 
198  error(compression);
199  }
200 
201  }; // class CompressionFactory
202 
203  class NoCompressor : public Compressor {
204 
205  int m_fd;
206 
207  public:
208 
209  NoCompressor(int fd, fsync sync) :
210  Compressor(sync),
211  m_fd(fd) {
212  }
213 
214  ~NoCompressor() noexcept final {
215  try {
216  close();
217  } catch (...) {
218  // Ignore any exceptions because destructor must not throw.
219  }
220  }
221 
222  void write(const std::string& data) final {
223  osmium::io::detail::reliable_write(m_fd, data.data(), data.size());
224  }
225 
226  void close() final {
227  if (m_fd >= 0) {
228  int fd = m_fd;
229  m_fd = -1;
230  if (do_fsync()) {
231  osmium::io::detail::reliable_fsync(fd);
232  }
233  osmium::io::detail::reliable_close(fd);
234  }
235  }
236 
237  }; // class NoCompressor
238 
239  class NoDecompressor : public Decompressor {
240 
241  int m_fd;
242  const char *m_buffer;
244 
245  public:
246 
247  explicit NoDecompressor(int fd) :
248  Decompressor(),
249  m_fd(fd),
250  m_buffer(nullptr),
251  m_buffer_size(0) {
252  }
253 
254  NoDecompressor(const char* buffer, size_t size) :
255  Decompressor(),
256  m_fd(-1),
257  m_buffer(buffer),
258  m_buffer_size(size) {
259  }
260 
261  ~NoDecompressor() noexcept final {
262  try {
263  close();
264  } catch (...) {
265  // Ignore any exceptions because destructor must not throw.
266  }
267  }
268 
269  std::string read() final {
270  std::string buffer;
271 
272  if (m_buffer) {
273  if (m_buffer_size != 0) {
274  size_t size = m_buffer_size;
275  m_buffer_size = 0;
276  buffer.append(m_buffer, size);
277  }
278  } else {
280  auto nread = ::read(m_fd, const_cast<char*>(buffer.data()), osmium::io::Decompressor::input_buffer_size);
281  if (nread < 0) {
282  throw std::system_error(errno, std::system_category(), "Read failed");
283  }
284  buffer.resize(std::string::size_type(nread));
285  }
286 
287  return buffer;
288  }
289 
290  void close() final {
291  if (m_fd >= 0) {
292  int fd = m_fd;
293  m_fd = -1;
294  osmium::io::detail::reliable_close(fd);
295  }
296  }
297 
298  }; // class NoDecompressor
299 
300  namespace detail {
301 
302  // we want the register_compression() function to run, setting
303  // the variable is only a side-effect, it will never be used
305  [](int fd, fsync sync) { return new osmium::io::NoCompressor(fd, sync); },
306  [](int fd) { return new osmium::io::NoDecompressor(fd); },
307  [](const char* buffer, size_t size) { return new osmium::io::NoDecompressor(buffer, size); }
308  );
309 
310  // dummy function to silence the unused variable warning from above
311  inline bool get_registered_no_compression() noexcept {
312  return registered_no_compression;
313  }
314 
315  } // namespace detail
316 
317  } // namespace io
318 
319 } // namespace osmium
320 
321 #endif // OSMIUM_IO_COMPRESSION_HPP
OSMIUM_NORETURN void error(osmium::io::file_compression compression)
Definition: compression.hpp:142
void close() final
Definition: compression.hpp:290
#define OSMIUM_NORETURN
Definition: compatibility.hpp:41
NoDecompressor(int fd)
Definition: compression.hpp:247
virtual std::string read()=0
Decompressor & operator=(const Decompressor &)=delete
std::string read() final
Definition: compression.hpp:269
static CompressionFactory & instance()
Definition: compression.hpp:151
static constexpr unsigned int input_buffer_size
Definition: compression.hpp:91
void close() final
Definition: compression.hpp:226
void write(const std::string &data) final
Definition: compression.hpp:222
~NoCompressor() noexceptfinal
Definition: compression.hpp:214
Definition: compression.hpp:87
fsync m_fsync
Definition: compression.hpp:64
virtual void write(const std::string &data)=0
Compressor(fsync sync)
Definition: compression.hpp:74
int m_fd
Definition: compression.hpp:241
const char * as_string(file_compression compression)
Definition: file_compression.hpp:51
size_t m_buffer_size
Definition: compression.hpp:243
Namespace for everything in the Osmium library.
Definition: assembler.hpp:59
Definition: attr.hpp:298
~NoDecompressor() noexceptfinal
Definition: compression.hpp:261
std::unique_ptr< osmium::io::Decompressor > create_decompressor(osmium::io::file_compression compression, int fd)
Definition: compression.hpp:181
fsync
Definition: writer_options.hpp:51
std::unique_ptr< osmium::io::Decompressor > create_decompressor(osmium::io::file_compression compression, const char *buffer, size_t size)
Definition: compression.hpp:191
virtual void close()=0
std::map< const osmium::io::file_compression, std::tuple< create_compressor_type, create_decompressor_type_fd, create_decompressor_type_buffer > > compression_map_type
Definition: compression.hpp:130
std::function< osmium::io::Decompressor *(int)> create_decompressor_type_fd
Definition: compression.hpp:122
bool do_fsync() const
Definition: compression.hpp:68
file_compression
Definition: file_compression.hpp:42
std::unique_ptr< osmium::io::Compressor > create_compressor(osmium::io::file_compression compression, TArgs &&...args)
Definition: compression.hpp:171
std::function< osmium::io::Decompressor *(const char *, size_t)> create_decompressor_type_buffer
Definition: compression.hpp:123
virtual ~Decompressor() noexcept
Definition: compression.hpp:101
Definition: compression.hpp:203
Definition: compression.hpp:62
int m_fd
Definition: compression.hpp:205
compression_map_type m_callbacks
Definition: compression.hpp:132
CompressionFactory & operator=(const CompressionFactory &)=delete
virtual ~Compressor() noexcept
Definition: compression.hpp:78
Definition: compression.hpp:239
Definition: compression.hpp:117
NoDecompressor(const char *buffer, size_t size)
Definition: compression.hpp:254
bool register_compression(osmium::io::file_compression compression, create_compressor_type create_compressor, create_decompressor_type_fd create_decompressor_fd, create_decompressor_type_buffer create_decompressor_buffer)
Definition: compression.hpp:156
std::function< osmium::io::Compressor *(int, fsync)> create_compressor_type
Definition: compression.hpp:121
const char * m_buffer
Definition: compression.hpp:242
NoCompressor(int fd, fsync sync)
Definition: compression.hpp:209
virtual void close()=0