Libosmium  2.4.1
Fast and flexible C++ library for working with OpenStreetMap data
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
buffer.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_MEMORY_BUFFER_HPP
2 #define OSMIUM_MEMORY_BUFFER_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstring>
40 #include <functional>
41 #include <iterator>
42 #include <stdexcept>
43 #include <utility>
44 #include <vector>
45 
46 #include <osmium/memory/item.hpp>
48 #include <osmium/osm/entity.hpp>
49 
50 namespace osmium {
51 
57  struct buffer_is_full : public std::runtime_error {
58 
60  std::runtime_error("Osmium buffer is full") {
61  }
62 
63  }; // struct buffer_is_full
64 
68  namespace memory {
69 
94  class Buffer {
95 
96  public:
97 
98  enum class auto_grow : bool {
99  yes = true,
100  no = false
101  }; // enum class auto_grow
102 
103  private:
104 
105  std::vector<unsigned char> m_memory;
106  unsigned char* m_data;
107  size_t m_capacity;
108  size_t m_written;
109  size_t m_committed;
111  std::function<void(Buffer&)> m_full;
112 
113  public:
114 
115  typedef Item value_type;
116 
122  Buffer() noexcept :
123  m_memory(),
124  m_data(nullptr),
125  m_capacity(0),
126  m_written(0),
127  m_committed(0) {
128  }
129 
138  explicit Buffer(unsigned char* data, size_t size) :
139  m_memory(),
140  m_data(data),
141  m_capacity(size),
142  m_written(size),
143  m_committed(size) {
144  if (size % align_bytes != 0) {
145  throw std::invalid_argument("buffer size needs to be multiple of alignment");
146  }
147  }
148 
158  explicit Buffer(unsigned char* data, size_t capacity, size_t committed) :
159  m_memory(),
160  m_data(data),
161  m_capacity(capacity),
162  m_written(committed),
163  m_committed(committed) {
164  if (capacity % align_bytes != 0) {
165  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
166  }
167  if (committed % align_bytes != 0) {
168  throw std::invalid_argument("buffer parameter 'committed' needs to be multiple of alignment");
169  }
170  }
171 
179  m_memory(capacity),
180  m_data(m_memory.data()),
181  m_capacity(capacity),
182  m_written(0),
183  m_committed(0),
185  if (capacity % align_bytes != 0) {
186  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
187  }
188  }
189 
190  // buffers can not be copied
191  Buffer(const Buffer&) = delete;
192  Buffer& operator=(const Buffer&) = delete;
193 
194  // buffers can be moved
195  Buffer(Buffer&&) = default;
196  Buffer& operator=(Buffer&&) = default;
197 
198  ~Buffer() = default;
199 
203  unsigned char* data() const noexcept {
204  return m_data;
205  }
206 
210  size_t capacity() const noexcept {
211  return m_capacity;
212  }
213 
217  size_t committed() const noexcept {
218  return m_committed;
219  }
220 
225  size_t written() const noexcept {
226  return m_written;
227  }
228 
233  bool is_aligned() const noexcept {
234  return (m_written % align_bytes == 0) && (m_committed % align_bytes == 0);
235  }
236 
241  void set_full_callback(std::function<void(Buffer&)> full) {
242  m_full = full;
243  }
244 
253  void grow(size_t size) {
254  if (m_memory.empty()) {
255  throw std::logic_error("Can't grow Buffer if it doesn't use internal memory management.");
256  }
257  if (m_capacity < size) {
258  if (size % align_bytes != 0) {
259  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
260  }
261  m_memory.resize(size);
262  m_data = m_memory.data();
263  m_capacity = size;
264  }
265  }
266 
272  size_t commit() {
273  assert(is_aligned());
274 
275  const size_t offset = m_committed;
276  m_committed = m_written;
277  return offset;
278  }
279 
283  void rollback() {
284  m_written = m_committed;
285  }
286 
292  size_t clear() {
293  const size_t committed = m_committed;
294  m_written = 0;
295  m_committed = 0;
296  return committed;
297  }
298 
305  template <class T>
306  T& get(const size_t offset) const {
307  return *reinterpret_cast<T*>(&m_data[offset]);
308  }
309 
335  unsigned char* reserve_space(const size_t size) {
336  if (m_written + size > m_capacity) {
337  if (m_full) {
338  m_full(*this);
339  } else if (!m_memory.empty() && (m_auto_grow == auto_grow::yes)) {
340  // double buffer size until there is enough space
341  size_t new_capacity = m_capacity * 2;
342  while (m_written + size > new_capacity) {
343  new_capacity *= 2;
344  }
345  grow(new_capacity);
346  } else {
347  throw osmium::buffer_is_full();
348  }
349  }
350  unsigned char* data = &m_data[m_written];
351  m_written += size;
352  return data;
353  }
354 
366  template <class T>
367  T& add_item(const T& item) {
368  unsigned char* target = reserve_space(item.padded_size());
369  std::copy_n(reinterpret_cast<const unsigned char*>(&item), item.padded_size(), target);
370  return *reinterpret_cast<T*>(target);
371  }
372 
379  void add_buffer(const Buffer& buffer) {
380  unsigned char* target = reserve_space(buffer.committed());
381  std::copy_n(reinterpret_cast<const unsigned char*>(buffer.data()), buffer.committed(), target);
382  }
383 
388  void push_back(const osmium::memory::Item& item) {
389  add_item(item);
390  commit();
391  }
392 
397  template <class T>
399 
400  template <class T>
402 
405 
406  template <class T>
408  return t_iterator<T>(m_data, m_data + m_committed);
409  }
410 
411  iterator begin() {
412  return iterator(m_data, m_data + m_committed);
413  }
414 
415  template <class T>
416  t_iterator<T> get_iterator(size_t offset) {
417  return t_iterator<T>(m_data + offset, m_data + m_committed);
418  }
419 
420  iterator get_iterator(size_t offset) {
421  return iterator(m_data + offset, m_data + m_committed);
422  }
423 
424  template <class T>
426  return t_iterator<T>(m_data + m_committed, m_data + m_committed);
427  }
428 
429  iterator end() {
430  return iterator(m_data + m_committed, m_data + m_committed);
431  }
432 
433  template <class T>
435  return t_const_iterator<T>(m_data, m_data + m_committed);
436  }
437 
438  const_iterator cbegin() const {
439  return const_iterator(m_data, m_data + m_committed);
440  }
441 
442  template <class T>
443  t_const_iterator<T> get_iterator(size_t offset) const {
444  return t_const_iterator<T>(m_data + offset, m_data + m_committed);
445  }
446 
447  const_iterator get_iterator(size_t offset) const {
448  return const_iterator(m_data + offset, m_data + m_committed);
449  }
450 
451  template <class T>
453  return t_const_iterator<T>(m_data + m_committed, m_data + m_committed);
454  }
455 
456  const_iterator cend() const {
457  return const_iterator(m_data + m_committed, m_data + m_committed);
458  }
459 
460  template <class T>
462  return cbegin<T>();
463  }
464 
465  const_iterator begin() const {
466  return cbegin();
467  }
468 
469  template <class T>
471  return cend<T>();
472  }
473 
474  const_iterator end() const {
475  return cend();
476  }
477 
481  explicit operator bool() const {
482  return m_data != nullptr;
483  }
484 
485  friend void swap(Buffer& lhs, Buffer& rhs) {
486  using std::swap;
487 
488  swap(lhs.m_memory, rhs.m_memory);
489  swap(lhs.m_data, rhs.m_data);
490  swap(lhs.m_capacity, rhs.m_capacity);
491  swap(lhs.m_written, rhs.m_written);
492  swap(lhs.m_committed, rhs.m_committed);
493  }
494 
509  template <class TCallbackClass>
510  void purge_removed(TCallbackClass* callback) {
511  if (begin() == end()) {
512  return;
513  }
514 
515  iterator it_write = begin();
516 
517  iterator next;
518  for (iterator it_read = begin(); it_read != end(); it_read = next) {
519  next = std::next(it_read);
520  if (!it_read->removed()) {
521  if (it_read != it_write) {
522  assert(it_read.data() >= data());
523  assert(it_write.data() >= data());
524  size_t old_offset = static_cast<size_t>(it_read.data() - data());
525  size_t new_offset = static_cast<size_t>(it_write.data() - data());
526  callback->moving_in_buffer(old_offset, new_offset);
527  std::memmove(it_write.data(), it_read.data(), it_read->padded_size());
528  }
529  it_write.advance_once();
530  }
531  }
532 
533  assert(it_write.data() >= data());
534  m_written = static_cast<size_t>(it_write.data() - data());
535  m_committed = m_written;
536  }
537 
538  }; // class Buffer
539 
540  inline bool operator==(const Buffer& lhs, const Buffer& rhs) noexcept {
541  return lhs.data() == rhs.data() && lhs.capacity() == rhs.capacity() && lhs.committed() == rhs.committed();
542  }
543 
544  inline bool operator!=(const Buffer& lhs, const Buffer& rhs) noexcept {
545  return ! (lhs == rhs);
546  }
547 
548  } // namespace memory
549 
550 } // namespace osmium
551 
552 #endif // OSMIUM_MEMORY_BUFFER_HPP
size_t m_written
Definition: buffer.hpp:108
Item value_type
Definition: buffer.hpp:115
t_const_iterator< T > cend() const
Definition: buffer.hpp:452
const_iterator begin() const
Definition: buffer.hpp:465
size_t clear()
Definition: buffer.hpp:292
bool is_aligned() const noexcept
Definition: buffer.hpp:233
size_t written() const noexcept
Definition: buffer.hpp:225
iterator get_iterator(size_t offset)
Definition: buffer.hpp:420
bool operator!=(const Buffer &lhs, const Buffer &rhs) noexcept
Definition: buffer.hpp:544
void grow(size_t size)
Definition: buffer.hpp:253
const_iterator end() const
Definition: buffer.hpp:474
Definition: item_iterator.hpp:131
unsigned char * m_data
Definition: buffer.hpp:106
t_const_iterator< osmium::OSMEntity > const_iterator
Definition: buffer.hpp:404
constexpr item_size_type align_bytes
Definition: item.hpp:53
Buffer(unsigned char *data, size_t capacity, size_t committed)
Definition: buffer.hpp:158
Definition: reader_iterator.hpp:39
ItemIterator< TMember > & advance_once()
Definition: item_iterator.hpp:180
Buffer(size_t capacity, auto_grow auto_grow=auto_grow::yes)
Definition: buffer.hpp:178
unsigned char * data() const
Definition: item_iterator.hpp:201
bool operator==(const Buffer &lhs, const Buffer &rhs) noexcept
Definition: buffer.hpp:540
t_iterator< T > end()
Definition: buffer.hpp:425
Definition: item.hpp:97
Namespace for everything in the Osmium library.
Definition: assembler.hpp:55
T & add_item(const T &item)
Definition: buffer.hpp:367
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:510
t_iterator< T > begin()
Definition: buffer.hpp:407
void add_buffer(const Buffer &buffer)
Definition: buffer.hpp:379
friend void swap(Buffer &lhs, Buffer &rhs)
Definition: buffer.hpp:485
const_iterator cbegin() const
Definition: buffer.hpp:438
size_t m_committed
Definition: buffer.hpp:109
unsigned char * data() const noexcept
Definition: buffer.hpp:203
Buffer() noexcept
Definition: buffer.hpp:122
size_t capacity() const noexcept
Definition: buffer.hpp:210
unsigned char * reserve_space(const size_t size)
Definition: buffer.hpp:335
auto_grow m_auto_grow
Definition: buffer.hpp:110
void push_back(const osmium::memory::Item &item)
Definition: buffer.hpp:388
iterator end()
Definition: buffer.hpp:429
iterator begin()
Definition: buffer.hpp:411
const_iterator cend() const
Definition: buffer.hpp:456
size_t m_capacity
Definition: buffer.hpp:107
t_const_iterator< T > get_iterator(size_t offset) const
Definition: buffer.hpp:443
size_t committed() const noexcept
Definition: buffer.hpp:217
Buffer(unsigned char *data, size_t size)
Definition: buffer.hpp:138
t_iterator< osmium::OSMEntity > iterator
Definition: buffer.hpp:403
t_const_iterator< T > end() const
Definition: buffer.hpp:470
Definition: buffer.hpp:94
Definition: buffer.hpp:57
const_iterator get_iterator(size_t offset) const
Definition: buffer.hpp:447
t_const_iterator< T > cbegin() const
Definition: buffer.hpp:434
t_const_iterator< T > begin() const
Definition: buffer.hpp:461
auto_grow
Definition: buffer.hpp:98
std::vector< unsigned char > m_memory
Definition: buffer.hpp:105
buffer_is_full()
Definition: buffer.hpp:59
void rollback()
Definition: buffer.hpp:283
t_iterator< T > get_iterator(size_t offset)
Definition: buffer.hpp:416
void set_full_callback(std::function< void(Buffer &)> full)
Definition: buffer.hpp:241
std::function< void(Buffer &)> m_full
Definition: buffer.hpp:111
Buffer & operator=(const Buffer &)=delete
size_t commit()
Definition: buffer.hpp:272