Libosmium  2.1.0
Fast and flexible C++ library for working with OpenStreetMap data
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
collector.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_RELATIONS_COLLECTOR_HPP
2 #define OSMIUM_RELATIONS_COLLECTOR_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstdint>
40 #include <functional>
41 #include <iomanip>
42 #include <iostream>
43 #include <vector>
44 
45 #include <osmium/osm/item_type.hpp>
46 #include <osmium/osm/object.hpp>
47 #include <osmium/osm/relation.hpp> // IWYU pragma: keep
48 #include <osmium/osm/types.hpp>
49 #include <osmium/handler.hpp>
50 #include <osmium/memory/buffer.hpp>
51 #include <osmium/visitor.hpp>
52 
53 #include <osmium/relations/detail/relation_meta.hpp>
54 #include <osmium/relations/detail/member_meta.hpp>
55 
56 namespace osmium {
57 
58  class Node;
59  class Way;
60 
64  namespace relations {
65 
94  template <class TCollector, bool TNodes, bool TWays, bool TRelations>
95  class Collector {
96 
101 
102  TCollector& m_collector;
103 
104  public:
105 
106  HandlerPass1(TCollector& collector) noexcept :
107  m_collector(collector) {
108  }
109 
111  if (m_collector.keep_relation(relation)) {
112  m_collector.add_relation(relation);
113  }
114  }
115 
116  }; // class HandlerPass1
117 
118  public:
119 
124 
125  TCollector& m_collector;
126 
139 
148  auto& mmv = m_collector.member_meta(object.type());
149  auto range = std::equal_range(mmv.begin(), mmv.end(), MemberMeta(object.id()));
150 
151  if (osmium::relations::count_not_removed(range.first, range.second) == 0) {
152  // nothing found
153  return false;
154  }
155 
156  {
157  m_collector.members_buffer().add_item(object);
158  const size_t member_offset = m_collector.members_buffer().commit();
159 
160  for (auto it = range.first; it != range.second; ++it) {
161  it->set_buffer_offset(member_offset);
162  }
163  }
164 
165  for (auto it = range.first; it != range.second; ++it) {
166  MemberMeta& member_meta = *it;
167  if (member_meta.removed()) {
168  break;
169  }
170  assert(member_meta.member_id() == object.id());
171  assert(member_meta.relation_pos() < m_collector.m_relations.size());
172  RelationMeta& relation_meta = m_collector.m_relations[member_meta.relation_pos()];
173 // std::cerr << " => " << member_meta.member_pos() << " < " << m_collector.get_relation(relation_meta).members().size() << " (id=" << m_collector.get_relation(relation_meta).id() << ")\n";
174  assert(member_meta.member_pos() < m_collector.get_relation(relation_meta).members().size());
175 // std::cerr << " add way " << member_meta.member_id() << " to rel " << m_collector.get_relation(relation_meta).id() << " at pos " << member_meta.member_pos() << "\n";
176  relation_meta.got_one_member();
177  if (relation_meta.has_all_members()) {
178  const size_t relation_offset = member_meta.relation_pos();
179  m_collector.complete_relation(relation_meta);
180  m_collector.m_relations[relation_offset] = RelationMeta();
181  m_collector.possibly_purge_removed_members();
182  }
183  }
184 
185  // Remove MemberMetas that were marked as removed.
186  mmv.erase(std::remove_if(mmv.begin(), mmv.end(), [](MemberMeta& mm) {
187  return mm.removed();
188  }), mmv.end());
189 
190  return true;
191  }
192 
193  public:
194 
195  HandlerPass2(TCollector& collector) noexcept :
196  m_collector(collector),
197  m_want_types((TNodes?1:0) + (TWays?1:0) + (TRelations?1:0)) {
198  }
199 
200  void node(const osmium::Node& node) {
201  if (TNodes) {
202  if (! find_and_add_object(node)) {
203  m_collector.node_not_in_any_relation(node);
204  }
205  }
206  }
207 
208  void way(const osmium::Way& way) {
209  if (TWays) {
210  if (! find_and_add_object(way)) {
211  m_collector.way_not_in_any_relation(way);
212  }
213  }
214  }
215 
217  if (TRelations) {
218  if (! find_and_add_object(relation)) {
219  m_collector.relation_not_in_any_relation(relation);
220  }
221  }
222  }
223 
224  void flush() {
225  m_collector.flush();
226  }
227 
228  }; // class HandlerPass2
229 
230  HandlerPass2 m_handler_pass2;
231 
232  // All relations we are interested in will be kept in this buffer
234 
235  // All members we are interested in will be kept in this buffer
237 
239  std::vector<RelationMeta> m_relations;
240 
245  std::vector<MemberMeta> m_member_meta[3];
246 
248 
249  typedef std::function<void(osmium::memory::Buffer&&)> callback_func_type;
250  callback_func_type m_callback;
251 
252  static constexpr size_t initial_buffer_size = 1024 * 1024;
253 
254  public:
255 
260  m_handler_pass2(*static_cast<TCollector*>(this)),
261  m_relations_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
262  m_members_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
263  m_relations(),
264  m_member_meta() {
265  }
266 
267  protected:
268 
269  std::vector<MemberMeta>& member_meta(const item_type type) {
270  return m_member_meta[static_cast<uint16_t>(type) - 1];
271  }
272 
273  callback_func_type callback() {
274  return m_callback;
275  }
276 
277  const std::vector<RelationMeta>& relations() const {
278  return m_relations;
279  }
280 
290  bool keep_relation(const osmium::Relation& /*relation*/) const {
291  return true;
292  }
293 
304  bool keep_member(const osmium::relations::RelationMeta& /*relation_meta*/, const osmium::RelationMember& /*member*/) const {
305  return true;
306  }
307 
315  void node_not_in_any_relation(const osmium::Node& /*node*/) {
316  }
317 
325  void way_not_in_any_relation(const osmium::Way& /*way*/) {
326  }
327 
335  void relation_not_in_any_relation(const osmium::Relation& /*relation*/) {
336  }
337 
349  void flush() {
350  }
351 
357  m_relations.erase(
358  std::remove_if(m_relations.begin(), m_relations.end(), has_all_members()),
359  m_relations.end()
360  );
361  }
362 
363  const osmium::Relation& get_relation(size_t offset) const {
364  return m_relations_buffer.get<osmium::Relation>(offset);
365  }
366 
370  const osmium::Relation& get_relation(const RelationMeta& relation_meta) const {
371  return get_relation(relation_meta.relation_offset());
372  }
373 
374  osmium::OSMObject& get_member(size_t offset) const {
375  return m_members_buffer.get<osmium::OSMObject>(offset);
376  }
377 
386  void add_relation(const osmium::Relation& relation) {
387  const size_t offset = m_relations_buffer.committed();
388  m_relations_buffer.add_item(relation);
389 
390  RelationMeta relation_meta(offset);
391 
392  int n=0;
393  for (auto& member : m_relations_buffer.get<osmium::Relation>(offset).members()) {
394  if (static_cast<TCollector*>(this)->keep_member(relation_meta, member)) {
395  member_meta(member.type()).emplace_back(member.ref(), m_relations.size(), n);
396  relation_meta.increment_need_members();
397  } else {
398  member.ref(0); // set member id to zero to indicate we are not interested
399  }
400  ++n;
401  }
402 
403  assert(offset == m_relations_buffer.committed());
404  if (relation_meta.has_all_members()) {
405  m_relations_buffer.rollback();
406  } else {
407  m_relations_buffer.commit();
408  m_relations.push_back(std::move(relation_meta));
409 // std::cerr << "added relation id=" << relation.id() << "\n";
410  }
411  }
412 
418 /* std::cerr << "relations: " << m_relations.size() << "\n";
419  std::cerr << "node members: " << m_member_meta[0].size() << "\n";
420  std::cerr << "way members: " << m_member_meta[1].size() << "\n";
421  std::cerr << "relation members: " << m_member_meta[2].size() << "\n";*/
422  std::sort(m_member_meta[0].begin(), m_member_meta[0].end());
423  std::sort(m_member_meta[1].begin(), m_member_meta[1].end());
424  std::sort(m_member_meta[2].begin(), m_member_meta[2].end());
425  }
426 
427  public:
428 
429  uint64_t used_memory() const {
430  const uint64_t nmembers = m_member_meta[0].capacity() + m_member_meta[1].capacity() + m_member_meta[2].capacity();
431  const uint64_t members = nmembers * sizeof(MemberMeta);
432  const uint64_t relations = m_relations.capacity() * sizeof(RelationMeta);
433  const uint64_t relations_buffer_capacity = m_relations_buffer.capacity();
434  const uint64_t members_buffer_capacity = m_members_buffer.capacity();
435 
436  std::cout << " nR = m_relations.capacity() ........... = " << std::setw(12) << m_relations.capacity() << "\n";
437  std::cout << " nMN = m_member_meta[NODE].capacity() ... = " << std::setw(12) << m_member_meta[0].capacity() << "\n";
438  std::cout << " nMW = m_member_meta[WAY].capacity() .... = " << std::setw(12) << m_member_meta[1].capacity() << "\n";
439  std::cout << " nMR = m_member_meta[RELATION].capacity() = " << std::setw(12) << m_member_meta[2].capacity() << "\n";
440  std::cout << " nM = m_member_meta[*].capacity() ...... = " << std::setw(12) << nmembers << "\n";
441 
442  std::cout << " sRM = sizeof(RelationMeta) ............. = " << std::setw(12) << sizeof(RelationMeta) << "\n";
443  std::cout << " sMM = sizeof(MemberMeta) ............... = " << std::setw(12) << sizeof(MemberMeta) << "\n\n";
444 
445  std::cout << " nR * sRM ............................... = " << std::setw(12) << relations << "\n";
446  std::cout << " nM * sMM ............................... = " << std::setw(12) << members << "\n";
447  std::cout << " relations_buffer_capacity .............. = " << std::setw(12) << relations_buffer_capacity << "\n";
448  std::cout << " members_buffer_capacity ................ = " << std::setw(12) << members_buffer_capacity << "\n";
449 
450  const uint64_t total = relations + members + relations_buffer_capacity + members_buffer_capacity;
451 
452  std::cout << " total .................................. = " << std::setw(12) << total << "\n";
453  std::cout << " =======================================================\n";
454 
455  return relations_buffer_capacity + members_buffer_capacity + relations + members;
456  }
457 
461  HandlerPass2& handler(const callback_func_type& callback = nullptr) {
462  m_callback = callback;
463  return m_handler_pass2;
464  }
465 
467  return m_members_buffer;
468  }
469 
471  const auto& mmv = member_meta(type);
472  const auto range = std::equal_range(mmv.cbegin(), mmv.cend(), MemberMeta(id));
473  assert(range.first != range.second);
474  return range.first->buffer_offset();
475  }
476 
477  template <class TIter>
478  void read_relations(TIter begin, TIter end) {
479  HandlerPass1 handler(*static_cast<TCollector*>(this));
480  osmium::apply(begin, end, handler);
482  }
483 
484  template <class TSource>
485  void read_relations(TSource& source) {
486  read_relations(std::begin(source), std::end(source));
487  source.close();
488  }
489 
490  void moving_in_buffer(size_t old_offset, size_t new_offset) {
491  const osmium::OSMObject& object = m_members_buffer.get<osmium::OSMObject>(old_offset);
492  auto& mmv = member_meta(object.type());
493  auto range = std::equal_range(mmv.begin(), mmv.end(), osmium::relations::MemberMeta(object.id()));
494  for (auto it = range.first; it != range.second; ++it) {
495  assert(it->buffer_offset() == old_offset);
496  it->set_buffer_offset(new_offset);
497  }
498  }
499 
508  if (m_count_complete > 10000) { // XXX
509  const size_t size_before = m_members_buffer.committed();
510  m_members_buffer.purge_removed(this);
511  const size_t size_after = m_members_buffer.committed();
512  double percent = static_cast<double>(size_before - size_after);
513  percent /= size_before;
514  percent *= 100;
515  std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
516  m_count_complete = 0;
517  }
518  }
519 
528  std::vector<const osmium::Relation*> get_incomplete_relations() const {
529  std::vector<const osmium::Relation*> relations;
530  for (const auto& relation_meta : m_relations) {
531  if (!relation_meta.has_all_members()) {
532  relations.push_back(&get_relation(relation_meta));
533  }
534  }
535  return relations;
536  }
537 
538  }; // class Collector
539 
540  } // namespace relations
541 
542 } // namespace osmium
543 
544 #endif // OSMIUM_RELATIONS_COLLECTOR_HPP
std::vector< const osmium::Relation * > get_incomplete_relations() const
Definition: collector.hpp:528
void relation(const osmium::Relation &relation)
Definition: collector.hpp:216
callback_func_type m_callback
Definition: collector.hpp:250
osmium::memory::Buffer & members_buffer()
Definition: collector.hpp:466
type
Definition: entity_bits.hpp:60
RelationMemberList & members()
Definition: relation.hpp:174
void way(const osmium::Way &way)
Definition: collector.hpp:208
item_type
Definition: item_type.hpp:42
void clean_assembled_relations()
Definition: collector.hpp:356
void moving_in_buffer(size_t old_offset, size_t new_offset)
Definition: collector.hpp:490
Definition: relation.hpp:162
static constexpr size_t initial_buffer_size
Definition: collector.hpp:252
Definition: handler.hpp:55
bool keep_member(const osmium::relations::RelationMeta &, const osmium::RelationMember &) const
Definition: collector.hpp:304
osmium::memory::Buffer m_members_buffer
Definition: collector.hpp:236
const std::vector< RelationMeta > & relations() const
Definition: collector.hpp:277
void read_relations(TSource &source)
Definition: collector.hpp:485
size_t get_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:470
void relation_not_in_any_relation(const osmium::Relation &)
Definition: collector.hpp:335
Definition: way.hpp:65
std::vector< MemberMeta > & member_meta(const item_type type)
Definition: collector.hpp:269
Definition: collector.hpp:123
HandlerPass2(TCollector &collector) noexcept
Definition: collector.hpp:195
std::vector< RelationMeta > m_relations
Vector with all relations we are interested in.
Definition: collector.hpp:239
void sort_member_meta()
Definition: collector.hpp:417
void apply(TIterator it, TIterator end, THandlers &...handlers)
Definition: visitor.hpp:236
iterator end()
Definition: object.hpp:337
void node_not_in_any_relation(const osmium::Node &)
Definition: collector.hpp:315
Definition: relation.hpp:54
TCollector & m_collector
Definition: collector.hpp:102
int64_t object_id_type
Type for OSM object (node, way, or relation) IDs.
Definition: types.hpp:46
Namespace for everything in the Osmium library.
Definition: assembler.hpp:55
T & add_item(const T &item)
Definition: buffer.hpp:368
Definition: collector.hpp:95
const osmium::Relation & get_relation(size_t offset) const
Definition: collector.hpp:363
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:493
HandlerPass2 m_handler_pass2
Definition: collector.hpp:230
Collector()
Definition: collector.hpp:259
std::function< void(osmium::memory::Buffer &&)> callback_func_type
Definition: collector.hpp:249
osmium::OSMObject & get_member(size_t offset) const
Definition: collector.hpp:374
size_t capacity() const noexcept
Definition: buffer.hpp:211
void relation(const osmium::Relation &relation)
Definition: collector.hpp:110
osmium::io::InputIterator< osmium::io::Reader > end(osmium::io::Reader &)
Definition: reader_iterator.hpp:45
Definition: collector.hpp:100
void flush()
Definition: collector.hpp:224
osmium::memory::Buffer m_relations_buffer
Definition: collector.hpp:233
uint64_t used_memory() const
Definition: collector.hpp:429
size_t committed() const noexcept
Definition: buffer.hpp:218
int m_want_types
Definition: collector.hpp:138
bool find_and_add_object(const osmium::OSMObject &object)
Definition: collector.hpp:147
callback_func_type callback()
Definition: collector.hpp:273
Definition: buffer.hpp:95
T & get(const size_t offset) const
Definition: buffer.hpp:307
void add_relation(const osmium::Relation &relation)
Definition: collector.hpp:386
HandlerPass2 & handler(const callback_func_type &callback=nullptr)
Definition: collector.hpp:461
void flush()
Definition: collector.hpp:349
Definition: node.hpp:47
void way_not_in_any_relation(const osmium::Way &)
Definition: collector.hpp:325
int m_count_complete
Definition: collector.hpp:247
void node(const osmium::Node &node)
Definition: collector.hpp:200
void possibly_purge_removed_members()
Definition: collector.hpp:506
bool keep_relation(const osmium::Relation &) const
Definition: collector.hpp:290
TCollector & m_collector
Definition: collector.hpp:125
void read_relations(TIter begin, TIter end)
Definition: collector.hpp:478
osmium::io::InputIterator< osmium::io::Reader > begin(osmium::io::Reader &reader)
Definition: reader_iterator.hpp:41
HandlerPass1(TCollector &collector) noexcept
Definition: collector.hpp:106
void rollback()
Definition: buffer.hpp:284
const osmium::Relation & get_relation(const RelationMeta &relation_meta) const
Definition: collector.hpp:370
Definition: object.hpp:57
size_t commit()
Definition: buffer.hpp:273
std::vector< MemberMeta > m_member_meta[3]
Definition: collector.hpp:245
iterator begin()
Definition: object.hpp:333