Libosmium  2.9.0
Fast and flexible C++ library for working with OpenStreetMap data
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
collector.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_RELATIONS_COLLECTOR_HPP
2 #define OSMIUM_RELATIONS_COLLECTOR_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstdint>
40 #include <functional>
41 #include <iomanip>
42 #include <iostream>
43 #include <vector>
44 
45 #include <osmium/osm/item_type.hpp>
46 #include <osmium/osm/object.hpp>
47 #include <osmium/osm/relation.hpp>
48 #include <osmium/osm/types.hpp>
49 #include <osmium/handler.hpp>
50 #include <osmium/memory/buffer.hpp>
51 #include <osmium/util/iterator.hpp>
52 #include <osmium/visitor.hpp>
53 
54 #include <osmium/relations/detail/relation_meta.hpp>
55 #include <osmium/relations/detail/member_meta.hpp>
56 
57 namespace osmium {
58 
59  class Node;
60  class Way;
61 
65  namespace relations {
66 
67  namespace detail {
68 
69  } // namespace detail
70 
99  template <typename TCollector, bool TNodes, bool TWays, bool TRelations>
100  class Collector {
101 
106 
107  TCollector& m_collector;
108 
109  public:
110 
111  explicit HandlerPass1(TCollector& collector) noexcept :
112  m_collector(collector) {
113  }
114 
116  if (m_collector.keep_relation(relation)) {
117  m_collector.add_relation(relation);
118  }
119  }
120 
121  }; // class HandlerPass1
122 
123  public:
124 
129 
130  TCollector& m_collector;
131 
132  public:
133 
134  explicit HandlerPass2(TCollector& collector) noexcept :
135  m_collector(collector) {
136  }
137 
138  void node(const osmium::Node& node) {
139  if (TNodes) {
140  if (! m_collector.find_and_add_object(node)) {
141  m_collector.node_not_in_any_relation(node);
142  }
143  }
144  }
145 
146  void way(const osmium::Way& way) {
147  if (TWays) {
148  if (! m_collector.find_and_add_object(way)) {
149  m_collector.way_not_in_any_relation(way);
150  }
151  }
152  }
153 
155  if (TRelations) {
156  if (! m_collector.find_and_add_object(relation)) {
157  m_collector.relation_not_in_any_relation(relation);
158  }
159  }
160  }
161 
162  void flush() {
163  m_collector.flush();
164  }
165 
166  }; // class HandlerPass2
167 
168  private:
169 
170  HandlerPass2 m_handler_pass2;
171 
172  // All relations we are interested in will be kept in this buffer
174 
175  // All members we are interested in will be kept in this buffer
177 
179  std::vector<RelationMeta> m_relations;
180 
185  using mm_vector_type = std::vector<MemberMeta>;
186  using mm_iterator = mm_vector_type::iterator;
188 
190 
191  using callback_func_type = std::function<void(osmium::memory::Buffer&&)>;
193 
194  static constexpr size_t initial_buffer_size = 1024 * 1024;
195 
197  auto& mmv = member_meta(type);
198  return make_range(std::equal_range(mmv.begin(), mmv.end(), MemberMeta(id)));
199  }
200 
201  public:
202 
207  m_handler_pass2(*static_cast<TCollector*>(this)),
208  m_relations_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
209  m_members_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
210  m_relations(),
211  m_member_meta() {
212  }
213 
214  protected:
215 
216  std::vector<MemberMeta>& member_meta(const item_type type) {
217  return m_member_meta[static_cast<uint16_t>(type) - 1];
218  }
219 
221  return m_callback;
222  }
223 
224  const std::vector<RelationMeta>& relations() const {
225  return m_relations;
226  }
227 
237  bool keep_relation(const osmium::Relation& /*relation*/) const {
238  return true;
239  }
240 
251  bool keep_member(const osmium::relations::RelationMeta& /*relation_meta*/, const osmium::RelationMember& /*member*/) const {
252  return true;
253  }
254 
262  void node_not_in_any_relation(const osmium::Node& /*node*/) {
263  }
264 
272  void way_not_in_any_relation(const osmium::Way& /*way*/) {
273  }
274 
282  void relation_not_in_any_relation(const osmium::Relation& /*relation*/) {
283  }
284 
296  void flush() {
297  }
298 
304  m_relations.erase(
305  std::remove_if(m_relations.begin(), m_relations.end(), has_all_members()),
306  m_relations.end()
307  );
308  }
309 
310  const osmium::Relation& get_relation(size_t offset) const {
311  assert(m_relations_buffer.committed() > offset);
312  return m_relations_buffer.get<osmium::Relation>(offset);
313  }
314 
318  const osmium::Relation& get_relation(const RelationMeta& relation_meta) const {
319  return get_relation(relation_meta.relation_offset());
320  }
321 
325  const osmium::Relation& get_relation(const MemberMeta& member_meta) const {
326  return get_relation(m_relations[member_meta.relation_pos()]);
327  }
328 
329  osmium::OSMObject& get_member(size_t offset) const {
330  assert(m_members_buffer.committed() > offset);
331  return m_members_buffer.get<osmium::OSMObject>(offset);
332  }
333 
334  private:
335 
344  void add_relation(const osmium::Relation& relation) {
345  const size_t offset = m_relations_buffer.committed();
346  m_relations_buffer.add_item(relation);
347 
348  RelationMeta relation_meta(offset);
349 
350  int n = 0;
351  for (auto& member : m_relations_buffer.get<osmium::Relation>(offset).members()) {
352  if (static_cast<TCollector*>(this)->keep_member(relation_meta, member)) {
353  member_meta(member.type()).emplace_back(member.ref(), m_relations.size(), n);
354  relation_meta.increment_need_members();
355  } else {
356  member.ref(0); // set member id to zero to indicate we are not interested
357  }
358  ++n;
359  }
360 
361  assert(offset == m_relations_buffer.committed());
362  if (relation_meta.has_all_members()) {
363  m_relations_buffer.rollback();
364  } else {
365  m_relations_buffer.commit();
366  m_relations.push_back(std::move(relation_meta));
367  }
368  }
369 
375  std::sort(m_member_meta[0].begin(), m_member_meta[0].end());
376  std::sort(m_member_meta[1].begin(), m_member_meta[1].end());
377  std::sort(m_member_meta[2].begin(), m_member_meta[2].end());
378  }
379 
381  return std::count_if(range.begin(), range.end(), [](MemberMeta& mm) {
382  return !mm.removed();
383  });
384  }
385 
394  auto range = find_member_meta(object.type(), object.id());
395 
396  if (count_not_removed(range) == 0) {
397  // nothing found
398  return false;
399  }
400 
401  {
402  members_buffer().add_item(object);
403  const size_t member_offset = members_buffer().commit();
404 
405  for (auto& member_meta : range) {
406  member_meta.set_buffer_offset(member_offset);
407  }
408  }
409 
410  for (auto& member_meta : range) {
411  if (member_meta.removed()) {
412  break;
413  }
414  assert(member_meta.member_id() == object.id());
415  assert(member_meta.relation_pos() < m_relations.size());
416  RelationMeta& relation_meta = m_relations[member_meta.relation_pos()];
417  assert(member_meta.member_pos() < get_relation(relation_meta).members().size());
418  relation_meta.got_one_member();
419  if (relation_meta.has_all_members()) {
420  const size_t relation_offset = member_meta.relation_pos();
421  static_cast<TCollector*>(this)->complete_relation(relation_meta);
422  clear_member_metas(relation_meta);
423  m_relations[relation_offset] = RelationMeta();
425  }
426  }
427 
428  return true;
429  }
430 
431  void clear_member_metas(const osmium::relations::RelationMeta& relation_meta) {
432  const osmium::Relation& relation = get_relation(relation_meta);
433  for (const auto& member : relation.members()) {
434  if (member.ref() != 0) {
435  const auto range = find_member_meta(member.type(), member.ref());
436  assert(!range.empty());
437 
438  // if this is the last time this object was needed
439  // then mark it as removed
440  if (count_not_removed(range) == 1) {
441  get_member(range.begin()->buffer_offset()).set_removed(true);
442  }
443 
444  for (auto& member_meta : range) {
445  if (!member_meta.removed() && relation.id() == get_relation(member_meta).id()) {
446  member_meta.remove();
447  break;
448  }
449  }
450  }
451  }
452  }
453 
454  public:
455 
456  uint64_t used_memory() const {
457  const uint64_t nmembers = m_member_meta[0].capacity() + m_member_meta[1].capacity() + m_member_meta[2].capacity();
458  const uint64_t members = nmembers * sizeof(MemberMeta);
459  const uint64_t relations = m_relations.capacity() * sizeof(RelationMeta);
460  const uint64_t relations_buffer_capacity = m_relations_buffer.capacity();
461  const uint64_t members_buffer_capacity = m_members_buffer.capacity();
462 
463  std::cerr << " nR = m_relations.capacity() ........... = " << std::setw(12) << m_relations.capacity() << "\n";
464  std::cerr << " nMN = m_member_meta[NODE].capacity() ... = " << std::setw(12) << m_member_meta[0].capacity() << "\n";
465  std::cerr << " nMW = m_member_meta[WAY].capacity() .... = " << std::setw(12) << m_member_meta[1].capacity() << "\n";
466  std::cerr << " nMR = m_member_meta[RELATION].capacity() = " << std::setw(12) << m_member_meta[2].capacity() << "\n";
467  std::cerr << " nM = m_member_meta[*].capacity() ...... = " << std::setw(12) << nmembers << "\n";
468 
469  std::cerr << " sRM = sizeof(RelationMeta) ............. = " << std::setw(12) << sizeof(RelationMeta) << "\n";
470  std::cerr << " sMM = sizeof(MemberMeta) ............... = " << std::setw(12) << sizeof(MemberMeta) << "\n\n";
471 
472  std::cerr << " nR * sRM ............................... = " << std::setw(12) << relations << "\n";
473  std::cerr << " nM * sMM ............................... = " << std::setw(12) << members << "\n";
474  std::cerr << " relations_buffer_capacity .............. = " << std::setw(12) << relations_buffer_capacity << "\n";
475  std::cerr << " members_buffer_capacity ................ = " << std::setw(12) << members_buffer_capacity << "\n";
476 
477  const uint64_t total = relations + members + relations_buffer_capacity + members_buffer_capacity;
478 
479  std::cerr << " total .................................. = " << std::setw(12) << total << "\n";
480  std::cerr << " =======================================================\n";
481 
482  return relations_buffer_capacity + members_buffer_capacity + relations + members;
483  }
484 
488  HandlerPass2& handler(const callback_func_type& callback = nullptr) {
489  m_callback = callback;
490  return m_handler_pass2;
491  }
492 
494  return m_members_buffer;
495  }
496 
498  const auto range = find_member_meta(type, id);
499  assert(!range.empty());
500  return range.begin()->buffer_offset();
501  }
502 
503  template <typename TIter>
504  void read_relations(TIter begin, TIter end) {
505  HandlerPass1 handler(*static_cast<TCollector*>(this));
506  osmium::apply(begin, end, handler);
508  }
509 
510  template <typename TSource>
511  void read_relations(TSource& source) {
512  read_relations(std::begin(source), std::end(source));
513  source.close();
514  }
515 
516  void moving_in_buffer(size_t old_offset, size_t new_offset) {
517  const osmium::OSMObject& object = m_members_buffer.get<osmium::OSMObject>(old_offset);
518  auto range = find_member_meta(object.type(), object.id());
519  for (auto& member_meta : range) {
520  assert(member_meta.buffer_offset() == old_offset);
521  member_meta.set_buffer_offset(new_offset);
522  }
523  }
524 
533  if (m_count_complete > 10000) { // XXX
534 // const size_t size_before = m_members_buffer.committed();
535  m_members_buffer.purge_removed(this);
536 /*
537  const size_t size_after = m_members_buffer.committed();
538  double percent = static_cast<double>(size_before - size_after);
539  percent /= size_before;
540  percent *= 100;
541  std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
542 */
543  m_count_complete = 0;
544  }
545  }
546 
555  std::vector<const osmium::Relation*> get_incomplete_relations() const {
556  std::vector<const osmium::Relation*> relations;
557  for (const auto& relation_meta : m_relations) {
558  if (!relation_meta.has_all_members()) {
559  relations.push_back(&get_relation(relation_meta));
560  }
561  }
562  return relations;
563  }
564 
565  }; // class Collector
566 
567  } // namespace relations
568 
569 } // namespace osmium
570 
571 #endif // OSMIUM_RELATIONS_COLLECTOR_HPP
std::vector< const osmium::Relation * > get_incomplete_relations() const
Definition: collector.hpp:555
void relation(const osmium::Relation &relation)
Definition: collector.hpp:154
void clear_member_metas(const osmium::relations::RelationMeta &relation_meta)
Definition: collector.hpp:431
Definition: iterator.hpp:43
callback_func_type m_callback
Definition: collector.hpp:192
osmium::memory::Buffer & members_buffer()
Definition: collector.hpp:493
type
Definition: entity_bits.hpp:63
iterator_range< It > make_range(P &&p)
Definition: iterator.hpp:77
RelationMemberList & members()
Definition: relation.hpp:177
mm_vector_type m_member_meta[3]
Definition: collector.hpp:187
void way(const osmium::Way &way)
Definition: collector.hpp:146
item_type
Definition: item_type.hpp:43
void clean_assembled_relations()
Definition: collector.hpp:303
void moving_in_buffer(size_t old_offset, size_t new_offset)
Definition: collector.hpp:516
Definition: relation.hpp:165
static constexpr size_t initial_buffer_size
Definition: collector.hpp:194
Definition: handler.hpp:71
bool keep_member(const osmium::relations::RelationMeta &, const osmium::RelationMember &) const
Definition: collector.hpp:251
osmium::memory::Buffer m_members_buffer
Definition: collector.hpp:176
const std::vector< RelationMeta > & relations() const
Definition: collector.hpp:224
void read_relations(TSource &source)
Definition: collector.hpp:511
size_t get_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:497
void relation_not_in_any_relation(const osmium::Relation &)
Definition: collector.hpp:282
Definition: way.hpp:66
std::vector< MemberMeta > & member_meta(const item_type type)
Definition: collector.hpp:216
bool find_and_add_object(const osmium::OSMObject &object)
Definition: collector.hpp:393
Definition: collector.hpp:128
HandlerPass2(TCollector &collector) noexcept
Definition: collector.hpp:134
std::vector< RelationMeta > m_relations
Vector with all relations we are interested in.
Definition: collector.hpp:179
void sort_member_meta()
Definition: collector.hpp:374
void apply(TIterator it, TIterator end, THandlers &...handlers)
Definition: visitor.hpp:234
void node_not_in_any_relation(const osmium::Node &)
Definition: collector.hpp:262
std::function< void(osmium::memory::Buffer &&)> callback_func_type
Definition: collector.hpp:191
Definition: relation.hpp:54
TCollector & m_collector
Definition: collector.hpp:107
Namespace for everything in the Osmium library.
Definition: assembler.hpp:73
T & add_item(const T &item)
Definition: buffer.hpp:457
Definition: collector.hpp:100
iterator_range< mm_iterator > find_member_meta(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:196
const osmium::Relation & get_relation(size_t offset) const
Definition: collector.hpp:310
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:700
Definition: attr.hpp:333
HandlerPass2 m_handler_pass2
Definition: collector.hpp:170
Collector()
Definition: collector.hpp:206
osmium::OSMObject & get_member(size_t offset) const
Definition: collector.hpp:329
size_t capacity() const noexcept
Definition: buffer.hpp:233
void relation(const osmium::Relation &relation)
Definition: collector.hpp:115
osmium::io::InputIterator< osmium::io::Reader > end(osmium::io::Reader &)
Definition: reader_iterator.hpp:45
Definition: collector.hpp:105
int64_t object_id_type
Type for OSM object (node, way, or relation) IDs.
Definition: types.hpp:45
void flush()
Definition: collector.hpp:162
std::vector< MemberMeta > mm_vector_type
Definition: collector.hpp:185
osmium::memory::Buffer m_relations_buffer
Definition: collector.hpp:173
object_id_type id() const noexcept
Get ID of this object.
Definition: object.hpp:110
uint64_t used_memory() const
Definition: collector.hpp:456
size_t committed() const noexcept
Definition: buffer.hpp:241
callback_func_type callback()
Definition: collector.hpp:220
Definition: buffer.hpp:97
T & get(const size_t offset) const
Definition: buffer.hpp:379
void add_relation(const osmium::Relation &relation)
Definition: collector.hpp:344
HandlerPass2 & handler(const callback_func_type &callback=nullptr)
Definition: collector.hpp:488
void flush()
Definition: collector.hpp:296
Definition: node.hpp:47
void way_not_in_any_relation(const osmium::Way &)
Definition: collector.hpp:272
int m_count_complete
Definition: collector.hpp:189
void node(const osmium::Node &node)
Definition: collector.hpp:138
void possibly_purge_removed_members()
Definition: collector.hpp:531
bool keep_relation(const osmium::Relation &) const
Definition: collector.hpp:237
const osmium::Relation & get_relation(const MemberMeta &member_meta) const
Definition: collector.hpp:325
It begin() const
Definition: iterator.hpp:59
TCollector & m_collector
Definition: collector.hpp:130
void read_relations(TIter begin, TIter end)
Definition: collector.hpp:504
osmium::io::InputIterator< osmium::io::Reader > begin(osmium::io::Reader &reader)
Definition: reader_iterator.hpp:41
HandlerPass1(TCollector &collector) noexcept
Definition: collector.hpp:111
void rollback()
Definition: buffer.hpp:349
const osmium::Relation & get_relation(const RelationMeta &relation_meta) const
Definition: collector.hpp:318
static iterator_range< mm_iterator >::iterator::difference_type count_not_removed(const iterator_range< mm_iterator > &range)
Definition: collector.hpp:380
It end() const
Definition: iterator.hpp:63
Definition: object.hpp:58
size_t commit()
Definition: buffer.hpp:335
size_type size() const noexcept
Definition: relation.hpp:158