Libosmium  2.15.2
Fast and flexible C++ library for working with OpenStreetMap data
collector.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_RELATIONS_COLLECTOR_HPP
2 #define OSMIUM_RELATIONS_COLLECTOR_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2019 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <osmium/handler.hpp>
38 #include <osmium/memory/buffer.hpp>
39 #include <osmium/osm/item_type.hpp>
40 #include <osmium/osm/object.hpp>
41 #include <osmium/osm/relation.hpp>
42 #include <osmium/osm/types.hpp>
43 #include <osmium/relations/detail/member_meta.hpp>
44 #include <osmium/relations/detail/relation_meta.hpp>
45 #include <osmium/util/iterator.hpp>
46 #include <osmium/visitor.hpp>
47 
48 #include <algorithm>
49 #include <array>
50 #include <cassert>
51 #include <cstddef>
52 #include <cstdint>
53 #include <functional>
54 #include <iomanip>
55 #include <iostream>
56 #include <utility>
57 #include <vector>
58 
59 namespace osmium {
60 
61  class Node;
62  class Way;
63 
67  namespace relations {
68 
97  template <typename TCollector, bool TNodes, bool TWays, bool TRelations>
98  class Collector {
99 
104 
105  TCollector& m_collector;
106 
107  public:
108 
109  explicit HandlerPass1(TCollector& collector) noexcept :
110  m_collector(collector) {
111  }
112 
114  if (m_collector.keep_relation(relation)) {
115  m_collector.add_relation(relation);
116  }
117  }
118 
119  }; // class HandlerPass1
120 
121  public:
122 
127 
129  TCollector& m_collector;
130 
131  public:
132 
133  explicit HandlerPass2(TCollector& collector) noexcept :
134  m_collector(collector) {
135  }
136 
137  void node(const osmium::Node& node) {
138  if (TNodes) {
139  m_check_order.node(node);
140  if (!m_collector.find_and_add_object(node)) {
141  m_collector.node_not_in_any_relation(node);
142  }
143  }
144  }
145 
146  void way(const osmium::Way& way) {
147  if (TWays) {
148  m_check_order.way(way);
149  if (!m_collector.find_and_add_object(way)) {
150  m_collector.way_not_in_any_relation(way);
151  }
152  }
153  }
154 
156  if (TRelations) {
157  m_check_order.relation(relation);
158  if (!m_collector.find_and_add_object(relation)) {
159  m_collector.relation_not_in_any_relation(relation);
160  }
161  }
162  }
163 
164  void flush() {
165  m_collector.flush();
166  }
167 
168  }; // class HandlerPass2
169 
170  private:
171 
172  HandlerPass2 m_handler_pass2;
173 
174  // All relations we are interested in will be kept in this buffer
176 
177  // All members we are interested in will be kept in this buffer
179 
181  std::vector<RelationMeta> m_relations;
182 
187  using mm_vector_type = std::vector<MemberMeta>;
188  using mm_iterator = mm_vector_type::iterator;
189  std::array<mm_vector_type, 3> m_member_meta;
190 
192 
193  using callback_func_type = std::function<void(osmium::memory::Buffer&&)>;
195 
196  enum {
197  initial_buffer_size = 1024UL * 1024UL
198  };
199 
201  auto& mmv = member_meta(type);
202  return make_range(std::equal_range(mmv.begin(), mmv.end(), MemberMeta(id)));
203  }
204 
205  public:
206 
211  m_handler_pass2(*static_cast<TCollector*>(this)),
212  m_relations_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
213  m_members_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes) {
214  }
215 
216  protected:
217 
218  std::vector<MemberMeta>& member_meta(const item_type type) {
219  return m_member_meta[static_cast<uint16_t>(type) - 1];
220  }
221 
223  return m_callback;
224  }
225 
226  const std::vector<RelationMeta>& relations() const {
227  return m_relations;
228  }
229 
239  bool keep_relation(const osmium::Relation& /*relation*/) const {
240  return true;
241  }
242 
253  bool keep_member(const RelationMeta& /*relation_meta*/, const osmium::RelationMember& /*member*/) const {
254  return true;
255  }
256 
264  void node_not_in_any_relation(const osmium::Node& /*node*/) {
265  }
266 
274  void way_not_in_any_relation(const osmium::Way& /*way*/) {
275  }
276 
284  void relation_not_in_any_relation(const osmium::Relation& /*relation*/) {
285  }
286 
298  void flush() {
299  }
300 
301  const osmium::Relation& get_relation(size_t offset) const {
302  assert(m_relations_buffer.committed() > offset);
303  return m_relations_buffer.get<osmium::Relation>(offset);
304  }
305 
309  const osmium::Relation& get_relation(const RelationMeta& relation_meta) const {
310  return get_relation(relation_meta.relation_offset());
311  }
312 
316  const osmium::Relation& get_relation(const MemberMeta& member_meta) const {
317  return get_relation(m_relations[member_meta.relation_pos()]);
318  }
319 
320  osmium::OSMObject& get_member(size_t offset) const {
321  assert(m_members_buffer.committed() > offset);
322  return m_members_buffer.get<osmium::OSMObject>(offset);
323  }
324 
325  private:
326 
336  const size_t offset = m_relations_buffer.committed();
337  m_relations_buffer.add_item(relation);
338 
339  RelationMeta relation_meta{offset};
340 
341  int n = 0;
342  for (auto& member : m_relations_buffer.get<osmium::Relation>(offset).members()) {
343  if (static_cast<TCollector*>(this)->keep_member(relation_meta, member)) {
344  member_meta(member.type()).emplace_back(member.ref(), m_relations.size(), n);
345  relation_meta.increment_need_members();
346  } else {
347  member.set_ref(0); // set member id to zero to indicate we are not interested
348  }
349  ++n;
350  }
351 
352  assert(offset == m_relations_buffer.committed());
353  if (relation_meta.has_all_members()) {
354  m_relations_buffer.rollback();
355  } else {
356  m_relations_buffer.commit();
357  m_relations.push_back(relation_meta);
358  }
359  }
360 
366  std::sort(m_member_meta[0].begin(), m_member_meta[0].end());
367  std::sort(m_member_meta[1].begin(), m_member_meta[1].end());
368  std::sort(m_member_meta[2].begin(), m_member_meta[2].end());
369  }
370 
372  return std::count_if(range.begin(), range.end(), [](MemberMeta& mm) {
373  return !mm.removed();
374  });
375  }
376 
385  auto range = find_member_meta(object.type(), object.id());
386 
387  if (count_not_removed(range) == 0) {
388  // nothing found
389  return false;
390  }
391 
392  {
393  members_buffer().add_item(object);
394  const size_t member_offset = members_buffer().commit();
395 
396  for (auto& member_meta : range) {
397  member_meta.set_buffer_offset(member_offset);
398  }
399  }
400 
401  for (auto& member_meta : range) {
402  if (member_meta.removed()) {
403  break;
404  }
405  assert(member_meta.member_id() == object.id());
406  assert(member_meta.relation_pos() < m_relations.size());
407  RelationMeta& relation_meta = m_relations[member_meta.relation_pos()];
408  assert(member_meta.member_pos() < get_relation(relation_meta).members().size());
409  relation_meta.got_one_member();
410  if (relation_meta.has_all_members()) {
411  const size_t relation_offset = member_meta.relation_pos();
412  static_cast<TCollector*>(this)->complete_relation(relation_meta);
413  clear_member_metas(relation_meta);
414  m_relations[relation_offset] = RelationMeta{};
416  }
417  }
418 
419  return true;
420  }
421 
422  void clear_member_metas(const RelationMeta& relation_meta) {
423  const osmium::Relation& relation = get_relation(relation_meta);
424  for (const auto& member : relation.members()) {
425  if (member.ref() != 0) {
426  const auto range = find_member_meta(member.type(), member.ref());
427  assert(!range.empty());
428 
429  // if this is the last time this object was needed
430  // then mark it as removed
431  if (count_not_removed(range) == 1) {
432  get_member(range.begin()->buffer_offset()).set_removed(true);
433  }
434 
435  for (auto& member_meta : range) {
436  if (!member_meta.removed() && relation.id() == get_relation(member_meta).id()) {
437  member_meta.remove();
438  break;
439  }
440  }
441  }
442  }
443  }
444 
445  public:
446 
447  uint64_t used_memory() const {
448  const uint64_t nmembers = m_member_meta[0].capacity() + m_member_meta[1].capacity() + m_member_meta[2].capacity();
449  const uint64_t members = nmembers * sizeof(MemberMeta);
450  const uint64_t relations = m_relations.capacity() * sizeof(RelationMeta);
451  const uint64_t relations_buffer_capacity = m_relations_buffer.capacity();
452  const uint64_t members_buffer_capacity = m_members_buffer.capacity();
453 
454  std::cerr << " nR = m_relations.capacity() ........... = " << std::setw(12) << m_relations.capacity() << "\n";
455  std::cerr << " nMN = m_member_meta[NODE].capacity() ... = " << std::setw(12) << m_member_meta[0].capacity() << "\n";
456  std::cerr << " nMW = m_member_meta[WAY].capacity() .... = " << std::setw(12) << m_member_meta[1].capacity() << "\n";
457  std::cerr << " nMR = m_member_meta[RELATION].capacity() = " << std::setw(12) << m_member_meta[2].capacity() << "\n";
458  std::cerr << " nM = m_member_meta[*].capacity() ...... = " << std::setw(12) << nmembers << "\n";
459 
460  std::cerr << " sRM = sizeof(RelationMeta) ............. = " << std::setw(12) << sizeof(RelationMeta) << "\n";
461  std::cerr << " sMM = sizeof(MemberMeta) ............... = " << std::setw(12) << sizeof(MemberMeta) << "\n\n";
462 
463  std::cerr << " nR * sRM ............................... = " << std::setw(12) << relations << "\n";
464  std::cerr << " nM * sMM ............................... = " << std::setw(12) << members << "\n";
465  std::cerr << " relations_buffer_capacity .............. = " << std::setw(12) << relations_buffer_capacity << "\n";
466  std::cerr << " members_buffer_capacity ................ = " << std::setw(12) << members_buffer_capacity << "\n";
467 
468  const uint64_t total = relations + members + relations_buffer_capacity + members_buffer_capacity;
469 
470  std::cerr << " total .................................. = " << std::setw(12) << total << "\n";
471  std::cerr << " =======================================================\n";
472 
473  return relations_buffer_capacity + members_buffer_capacity + relations + members;
474  }
475 
479  HandlerPass2& handler(const callback_func_type& callback = nullptr) {
480  m_callback = callback;
481  return m_handler_pass2;
482  }
483 
485  return m_members_buffer;
486  }
487 
500  const auto range = find_member_meta(type, id);
501  assert(!range.empty());
502  return range.begin()->is_available();
503  }
504 
515  const auto range = find_member_meta(type, id);
516  assert(!range.empty());
517  assert(range.begin()->is_available());
518  return range.begin()->buffer_offset();
519  }
520 
538  const auto range = find_member_meta(type, id);
539  assert(!range.empty());
540  if (range.begin()->is_available()) {
541  return std::make_pair(true, range.begin()->buffer_offset());
542  }
543  return std::make_pair(false, 0);
544  }
545 
546  template <typename TIter>
547  void read_relations(TIter begin, TIter end) {
548  HandlerPass1 handler(*static_cast<TCollector*>(this));
549  osmium::apply(begin, end, handler);
551  }
552 
553  template <typename TSource>
554  void read_relations(TSource& source) {
555  using std::begin;
556  using std::end;
557  read_relations(begin(source), end(source));
558  source.close();
559  }
560 
561  void moving_in_buffer(size_t old_offset, size_t new_offset) {
562  const osmium::OSMObject& object = m_members_buffer.get<osmium::OSMObject>(old_offset);
563  auto range = find_member_meta(object.type(), object.id());
564  for (auto& member_meta : range) {
565  assert(member_meta.buffer_offset() == old_offset);
566  member_meta.set_buffer_offset(new_offset);
567  }
568  }
569 
578  if (m_count_complete > 10000) { // XXX
579 // const size_t size_before = m_members_buffer.committed();
580  m_members_buffer.purge_removed(this);
581 /*
582  const size_t size_after = m_members_buffer.committed();
583  double percent = static_cast<double>(size_before - size_after);
584  percent /= size_before;
585  percent *= 100;
586  std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
587 */
588  m_count_complete = 0;
589  }
590  }
591 
600  std::vector<const osmium::Relation*> get_incomplete_relations() const {
601  std::vector<const osmium::Relation*> relations;
602  for (const auto& relation_meta : m_relations) {
603  if (!relation_meta.has_all_members()) {
604  relations.push_back(&get_relation(relation_meta));
605  }
606  }
607  return relations;
608  }
609 
610  }; // class Collector
611 
612  } // namespace relations
613 
614 } // namespace osmium
615 
616 #endif // OSMIUM_RELATIONS_COLLECTOR_HPP
void relation(const osmium::Relation &relation)
Definition: collector.hpp:155
std::size_t capacity() const noexcept
Definition: buffer.hpp:348
Definition: iterator.hpp:42
bool keep_member(const RelationMeta &, const osmium::RelationMember &) const
Definition: collector.hpp:253
callback_func_type m_callback
Definition: collector.hpp:194
std::size_t commit()
Definition: buffer.hpp:468
osmium::memory::Buffer & members_buffer()
Definition: collector.hpp:484
osmium::OSMObject & get_member(size_t offset) const
Definition: collector.hpp:320
type
Definition: entity_bits.hpp:63
void way(const osmium::Way &) const noexcept
Definition: handler.hpp:81
RelationMemberList & members()
Get a reference to the member list.
Definition: relation.hpp:186
bool is_available(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:499
const std::vector< RelationMeta > & relations() const
Definition: collector.hpp:226
Definition: check_order.hpp:87
void node(const osmium::Node &) const noexcept
Definition: handler.hpp:78
bool keep_relation(const osmium::Relation &) const
Definition: collector.hpp:239
void way(const osmium::Way &way)
Definition: collector.hpp:146
item_type
Definition: item_type.hpp:43
std::vector< const osmium::Relation * > get_incomplete_relations() const
Definition: collector.hpp:600
InputIterator< Reader > end(Reader &)
Definition: reader_iterator.hpp:47
void moving_in_buffer(size_t old_offset, size_t new_offset)
Definition: collector.hpp:561
Definition: relation.hpp:168
void node(const osmium::Node &node)
Definition: check_order.hpp:95
Definition: handler.hpp:71
iterator_range< It > make_range(P &&p) noexcept
Definition: iterator.hpp:68
size_type size() const noexcept
Definition: collection.hpp:152
osmium::memory::Buffer m_members_buffer
Definition: collector.hpp:178
void read_relations(TSource &source)
Definition: collector.hpp:554
size_t get_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:514
void relation_not_in_any_relation(const osmium::Relation &)
Definition: collector.hpp:284
Definition: way.hpp:72
std::vector< MemberMeta > & member_meta(const item_type type)
Definition: collector.hpp:218
bool find_and_add_object(const osmium::OSMObject &object)
Definition: collector.hpp:384
Definition: collector.hpp:126
HandlerPass2(TCollector &collector) noexcept
Definition: collector.hpp:133
std::vector< RelationMeta > m_relations
Vector with all relations we are interested in.
Definition: collector.hpp:181
std::pair< bool, size_t > get_availability_and_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:537
std::size_t committed() const noexcept
Definition: buffer.hpp:356
void sort_member_meta()
Definition: collector.hpp:365
void node_not_in_any_relation(const osmium::Node &)
Definition: collector.hpp:264
void way(const osmium::Way &way)
Definition: check_order.hpp:112
InputIterator< Reader > begin(Reader &reader)
Definition: reader_iterator.hpp:43
std::function< void(osmium::memory::Buffer &&)> callback_func_type
Definition: collector.hpp:193
Definition: relation.hpp:57
TCollector & m_collector
Definition: collector.hpp:105
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
T & add_item(const T &item)
Definition: buffer.hpp:601
Definition: collector.hpp:98
std::array< mm_vector_type, 3 > m_member_meta
Definition: collector.hpp:189
iterator_range< mm_iterator > find_member_meta(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:200
It end() const noexcept
Definition: iterator.hpp:54
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:854
T & get(const std::size_t offset) const
Definition: buffer.hpp:518
HandlerPass2 m_handler_pass2
Definition: collector.hpp:172
Collector()
Definition: collector.hpp:210
void apply(TIterator it, TIterator end, THandlers &&... handlers)
Definition: visitor.hpp:323
const osmium::Relation & get_relation(const RelationMeta &relation_meta) const
Definition: collector.hpp:309
uint64_t used_memory() const
Definition: collector.hpp:447
void relation(const osmium::Relation &relation)
Definition: collector.hpp:113
Definition: collector.hpp:103
int64_t object_id_type
Type for OSM object (node, way, or relation) IDs.
Definition: types.hpp:45
void flush()
Definition: collector.hpp:164
std::vector< MemberMeta > mm_vector_type
Definition: collector.hpp:187
It begin() const noexcept
Definition: iterator.hpp:50
osmium::memory::Buffer m_relations_buffer
Definition: collector.hpp:175
void clear_member_metas(const RelationMeta &relation_meta)
Definition: collector.hpp:422
object_id_type id() const noexcept
Get ID of this object.
Definition: object.hpp:122
void relation(const osmium::Relation &relation)
Definition: check_order.hpp:126
osmium::handler::CheckOrder m_check_order
Definition: collector.hpp:128
callback_func_type callback()
Definition: collector.hpp:222
Definition: buffer.hpp:97
void add_relation(const osmium::Relation &relation)
Definition: collector.hpp:335
HandlerPass2 & handler(const callback_func_type &callback=nullptr)
Definition: collector.hpp:479
void flush()
Definition: collector.hpp:298
Definition: node.hpp:48
void way_not_in_any_relation(const osmium::Way &)
Definition: collector.hpp:274
int m_count_complete
Definition: collector.hpp:191
void node(const osmium::Node &node)
Definition: collector.hpp:137
void possibly_purge_removed_members()
Definition: collector.hpp:576
TCollector & m_collector
Definition: collector.hpp:129
void read_relations(TIter begin, TIter end)
Definition: collector.hpp:547
const osmium::Relation & get_relation(size_t offset) const
Definition: collector.hpp:301
HandlerPass1(TCollector &collector) noexcept
Definition: collector.hpp:109
void rollback()
Definition: buffer.hpp:484
static iterator_range< mm_iterator >::iterator::difference_type count_not_removed(const iterator_range< mm_iterator > &range)
Definition: collector.hpp:371
const osmium::Relation & get_relation(const MemberMeta &member_meta) const
Definition: collector.hpp:316
Definition: object.hpp:64