Libosmium  2.2.0
Fast and flexible C++ library for working with OpenStreetMap data
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
buffer.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_MEMORY_BUFFER_HPP
2 #define OSMIUM_MEMORY_BUFFER_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstring>
40 #include <exception>
41 #include <functional>
42 #include <iterator>
43 #include <stdexcept>
44 #include <utility>
45 #include <vector>
46 
47 #include <osmium/memory/item.hpp>
49 #include <osmium/osm/entity.hpp>
50 
51 namespace osmium {
52 
58  struct buffer_is_full : public std::runtime_error {
59 
61  std::runtime_error("Osmium buffer is full") {
62  }
63 
64  }; // struct buffer_is_full
65 
69  namespace memory {
70 
95  class Buffer {
96 
97  public:
98 
99  enum class auto_grow : bool {
100  yes = true,
101  no = false
102  }; // enum class auto_grow
103 
104  private:
105 
106  std::vector<unsigned char> m_memory;
107  unsigned char* m_data;
108  size_t m_capacity;
109  size_t m_written;
110  size_t m_committed;
112  std::function<void(Buffer&)> m_full;
113 
114  public:
115 
116  typedef Item value_type;
117 
123  Buffer() noexcept :
124  m_memory(),
125  m_data(nullptr),
126  m_capacity(0),
127  m_written(0),
128  m_committed(0) {
129  }
130 
139  explicit Buffer(unsigned char* data, size_t size) :
140  m_memory(),
141  m_data(data),
142  m_capacity(size),
143  m_written(size),
144  m_committed(size) {
145  if (size % align_bytes != 0) {
146  throw std::invalid_argument("buffer size needs to be multiple of alignment");
147  }
148  }
149 
159  explicit Buffer(unsigned char* data, size_t capacity, size_t committed) :
160  m_memory(),
161  m_data(data),
162  m_capacity(capacity),
163  m_written(committed),
164  m_committed(committed) {
165  if (capacity % align_bytes != 0) {
166  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
167  }
168  if (committed % align_bytes != 0) {
169  throw std::invalid_argument("buffer parameter 'committed' needs to be multiple of alignment");
170  }
171  }
172 
180  m_memory(capacity),
181  m_data(m_memory.data()),
182  m_capacity(capacity),
183  m_written(0),
184  m_committed(0),
186  if (capacity % align_bytes != 0) {
187  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
188  }
189  }
190 
191  // buffers can not be copied
192  Buffer(const Buffer&) = delete;
193  Buffer& operator=(const Buffer&) = delete;
194 
195  // buffers can be moved
196  Buffer(Buffer&&) = default;
197  Buffer& operator=(Buffer&&) = default;
198 
199  ~Buffer() = default;
200 
204  unsigned char* data() const noexcept {
205  return m_data;
206  }
207 
211  size_t capacity() const noexcept {
212  return m_capacity;
213  }
214 
218  size_t committed() const noexcept {
219  return m_committed;
220  }
221 
226  size_t written() const noexcept {
227  return m_written;
228  }
229 
234  bool is_aligned() const noexcept {
235  return (m_written % align_bytes == 0) && (m_committed % align_bytes == 0);
236  }
237 
242  void set_full_callback(std::function<void(Buffer&)> full) {
243  m_full = full;
244  }
245 
254  void grow(size_t size) {
255  if (m_memory.empty()) {
256  throw std::logic_error("Can't grow Buffer if it doesn't use internal memory management.");
257  }
258  if (m_capacity < size) {
259  if (size % align_bytes != 0) {
260  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
261  }
262  m_memory.resize(size);
263  m_data = m_memory.data();
264  m_capacity = size;
265  }
266  }
267 
273  size_t commit() {
274  assert(is_aligned());
275 
276  const size_t offset = m_committed;
277  m_committed = m_written;
278  return offset;
279  }
280 
284  void rollback() {
285  m_written = m_committed;
286  }
287 
293  size_t clear() {
294  const size_t committed = m_committed;
295  m_written = 0;
296  m_committed = 0;
297  return committed;
298  }
299 
306  template <class T>
307  T& get(const size_t offset) const {
308  return *reinterpret_cast<T*>(&m_data[offset]);
309  }
310 
336  unsigned char* reserve_space(const size_t size) {
337  if (m_written + size > m_capacity) {
338  if (m_full) {
339  m_full(*this);
340  } else if (!m_memory.empty() && (m_auto_grow == auto_grow::yes)) {
341  // double buffer size until there is enough space
342  size_t new_capacity = m_capacity * 2;
343  while (m_written + size > new_capacity) {
344  new_capacity *= 2;
345  }
346  grow(new_capacity);
347  } else {
348  throw osmium::buffer_is_full();
349  }
350  }
351  unsigned char* data = &m_data[m_written];
352  m_written += size;
353  return data;
354  }
355 
367  template <class T>
368  T& add_item(const T& item) {
369  unsigned char* target = reserve_space(item.padded_size());
370  std::copy_n(reinterpret_cast<const unsigned char*>(&item), item.padded_size(), target);
371  return *reinterpret_cast<T*>(target);
372  }
373 
380  void add_buffer(const Buffer& buffer) {
381  unsigned char* target = reserve_space(buffer.committed());
382  std::copy_n(reinterpret_cast<const unsigned char*>(buffer.data()), buffer.committed(), target);
383  }
384 
389  void push_back(const osmium::memory::Item& item) {
390  add_item(item);
391  commit();
392  }
393 
398  template <class T>
400 
401  template <class T>
403 
406 
407  template <class T>
409  return t_iterator<T>(m_data, m_data + m_committed);
410  }
411 
412  iterator begin() {
413  return iterator(m_data, m_data + m_committed);
414  }
415 
416  template <class T>
417  t_iterator<T> get_iterator(size_t offset) {
418  return t_iterator<T>(m_data + offset, m_data + m_committed);
419  }
420 
421  iterator get_iterator(size_t offset) {
422  return iterator(m_data + offset, m_data + m_committed);
423  }
424 
425  template <class T>
427  return t_iterator<T>(m_data + m_committed, m_data + m_committed);
428  }
429 
430  iterator end() {
431  return iterator(m_data + m_committed, m_data + m_committed);
432  }
433 
434  template <class T>
436  return t_const_iterator<T>(m_data, m_data + m_committed);
437  }
438 
439  const_iterator cbegin() const {
440  return const_iterator(m_data, m_data + m_committed);
441  }
442 
443  template <class T>
444  t_const_iterator<T> get_iterator(size_t offset) const {
445  return t_const_iterator<T>(m_data + offset, m_data + m_committed);
446  }
447 
448  const_iterator get_iterator(size_t offset) const {
449  return const_iterator(m_data + offset, m_data + m_committed);
450  }
451 
452  template <class T>
454  return t_const_iterator<T>(m_data + m_committed, m_data + m_committed);
455  }
456 
457  const_iterator cend() const {
458  return const_iterator(m_data + m_committed, m_data + m_committed);
459  }
460 
461  template <class T>
463  return cbegin<T>();
464  }
465 
466  const_iterator begin() const {
467  return cbegin();
468  }
469 
470  template <class T>
472  return cend<T>();
473  }
474 
475  const_iterator end() const {
476  return cend();
477  }
478 
482  explicit operator bool() const {
483  return m_data != nullptr;
484  }
485 
486  friend void swap(Buffer& lhs, Buffer& rhs) {
487  using std::swap;
488 
489  swap(lhs.m_memory, rhs.m_memory);
490  swap(lhs.m_data, rhs.m_data);
491  swap(lhs.m_capacity, rhs.m_capacity);
492  swap(lhs.m_written, rhs.m_written);
493  swap(lhs.m_committed, rhs.m_committed);
494  }
495 
510  template <class TCallbackClass>
511  void purge_removed(TCallbackClass* callback) {
512  if (begin() == end()) {
513  return;
514  }
515 
516  iterator it_write = begin();
517 
518  iterator next;
519  for (iterator it_read = begin(); it_read != end(); it_read = next) {
520  next = std::next(it_read);
521  if (!it_read->removed()) {
522  if (it_read != it_write) {
523  assert(it_read.data() >= data());
524  assert(it_write.data() >= data());
525  size_t old_offset = static_cast<size_t>(it_read.data() - data());
526  size_t new_offset = static_cast<size_t>(it_write.data() - data());
527  callback->moving_in_buffer(old_offset, new_offset);
528  std::memmove(it_write.data(), it_read.data(), it_read->padded_size());
529  }
530  it_write.advance_once();
531  }
532  }
533 
534  assert(it_write.data() >= data());
535  m_written = static_cast<size_t>(it_write.data() - data());
536  m_committed = m_written;
537  }
538 
539  }; // class Buffer
540 
541  inline bool operator==(const Buffer& lhs, const Buffer& rhs) noexcept {
542  return lhs.data() == rhs.data() && lhs.capacity() == rhs.capacity() && lhs.committed() == rhs.committed();
543  }
544 
545  inline bool operator!=(const Buffer& lhs, const Buffer& rhs) noexcept {
546  return ! (lhs == rhs);
547  }
548 
549  } // namespace memory
550 
551 } // namespace osmium
552 
553 #endif // OSMIUM_MEMORY_BUFFER_HPP
size_t m_written
Definition: buffer.hpp:109
Item value_type
Definition: buffer.hpp:116
t_const_iterator< T > cend() const
Definition: buffer.hpp:453
const_iterator begin() const
Definition: buffer.hpp:466
size_t clear()
Definition: buffer.hpp:293
bool is_aligned() const noexcept
Definition: buffer.hpp:234
size_t written() const noexcept
Definition: buffer.hpp:226
iterator get_iterator(size_t offset)
Definition: buffer.hpp:421
bool operator!=(const Buffer &lhs, const Buffer &rhs) noexcept
Definition: buffer.hpp:545
void grow(size_t size)
Definition: buffer.hpp:254
const_iterator end() const
Definition: buffer.hpp:475
Definition: item_iterator.hpp:131
unsigned char * m_data
Definition: buffer.hpp:107
t_const_iterator< osmium::OSMEntity > const_iterator
Definition: buffer.hpp:405
constexpr item_size_type align_bytes
Definition: item.hpp:54
Buffer(unsigned char *data, size_t capacity, size_t committed)
Definition: buffer.hpp:159
Definition: reader_iterator.hpp:39
ItemIterator< TMember > & advance_once()
Definition: item_iterator.hpp:180
Buffer(size_t capacity, auto_grow auto_grow=auto_grow::yes)
Definition: buffer.hpp:179
unsigned char * data() const
Definition: item_iterator.hpp:201
bool operator==(const Buffer &lhs, const Buffer &rhs) noexcept
Definition: buffer.hpp:541
t_iterator< T > end()
Definition: buffer.hpp:426
Definition: item.hpp:98
Namespace for everything in the Osmium library.
Definition: assembler.hpp:55
T & add_item(const T &item)
Definition: buffer.hpp:368
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:511
t_iterator< T > begin()
Definition: buffer.hpp:408
void add_buffer(const Buffer &buffer)
Definition: buffer.hpp:380
friend void swap(Buffer &lhs, Buffer &rhs)
Definition: buffer.hpp:486
const_iterator cbegin() const
Definition: buffer.hpp:439
size_t m_committed
Definition: buffer.hpp:110
unsigned char * data() const noexcept
Definition: buffer.hpp:204
Buffer() noexcept
Definition: buffer.hpp:123
size_t capacity() const noexcept
Definition: buffer.hpp:211
unsigned char * reserve_space(const size_t size)
Definition: buffer.hpp:336
auto_grow m_auto_grow
Definition: buffer.hpp:111
void push_back(const osmium::memory::Item &item)
Definition: buffer.hpp:389
iterator end()
Definition: buffer.hpp:430
iterator begin()
Definition: buffer.hpp:412
const_iterator cend() const
Definition: buffer.hpp:457
size_t m_capacity
Definition: buffer.hpp:108
t_const_iterator< T > get_iterator(size_t offset) const
Definition: buffer.hpp:444
size_t committed() const noexcept
Definition: buffer.hpp:218
Buffer(unsigned char *data, size_t size)
Definition: buffer.hpp:139
t_iterator< osmium::OSMEntity > iterator
Definition: buffer.hpp:404
t_const_iterator< T > end() const
Definition: buffer.hpp:471
Definition: buffer.hpp:95
Definition: buffer.hpp:58
const_iterator get_iterator(size_t offset) const
Definition: buffer.hpp:448
t_const_iterator< T > cbegin() const
Definition: buffer.hpp:435
t_const_iterator< T > begin() const
Definition: buffer.hpp:462
auto_grow
Definition: buffer.hpp:99
std::vector< unsigned char > m_memory
Definition: buffer.hpp:106
buffer_is_full()
Definition: buffer.hpp:60
void rollback()
Definition: buffer.hpp:284
t_iterator< T > get_iterator(size_t offset)
Definition: buffer.hpp:417
void set_full_callback(std::function< void(Buffer &)> full)
Definition: buffer.hpp:242
std::function< void(Buffer &)> m_full
Definition: buffer.hpp:112
Buffer & operator=(const Buffer &)=delete
size_t commit()
Definition: buffer.hpp:273