Libosmium  2.6.0
Fast and flexible C++ library for working with OpenStreetMap data
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
collector.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_RELATIONS_COLLECTOR_HPP
2 #define OSMIUM_RELATIONS_COLLECTOR_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstdint>
40 #include <functional>
41 #include <iomanip>
42 //#include <iostream>
43 #include <vector>
44 
45 #include <osmium/fwd.hpp>
46 #include <osmium/osm/item_type.hpp>
47 #include <osmium/osm/object.hpp>
48 #include <osmium/osm/relation.hpp> // IWYU pragma: keep
49 #include <osmium/osm/types.hpp>
50 #include <osmium/handler.hpp>
51 #include <osmium/memory/buffer.hpp>
52 #include <osmium/visitor.hpp>
53 
54 #include <osmium/relations/detail/relation_meta.hpp>
55 #include <osmium/relations/detail/member_meta.hpp>
56 
57 namespace osmium {
58 
62  namespace relations {
63 
92  template <typename TCollector, bool TNodes, bool TWays, bool TRelations>
93  class Collector {
94 
99 
100  TCollector& m_collector;
101 
102  public:
103 
104  HandlerPass1(TCollector& collector) noexcept :
105  m_collector(collector) {
106  }
107 
109  if (m_collector.keep_relation(relation)) {
110  m_collector.add_relation(relation);
111  }
112  }
113 
114  }; // class HandlerPass1
115 
116  public:
117 
122 
123  TCollector& m_collector;
124 
125  public:
126 
127  HandlerPass2(TCollector& collector) noexcept :
128  m_collector(collector) {
129  }
130 
131  void node(const osmium::Node& node) {
132  if (TNodes) {
133  if (! m_collector.find_and_add_object(node)) {
134  m_collector.node_not_in_any_relation(node);
135  }
136  }
137  }
138 
139  void way(const osmium::Way& way) {
140  if (TWays) {
141  if (! m_collector.find_and_add_object(way)) {
142  m_collector.way_not_in_any_relation(way);
143  }
144  }
145  }
146 
148  if (TRelations) {
149  if (! m_collector.find_and_add_object(relation)) {
150  m_collector.relation_not_in_any_relation(relation);
151  }
152  }
153  }
154 
155  void flush() {
156  m_collector.flush();
157  }
158 
159  }; // class HandlerPass2
160 
161  private:
162 
163  HandlerPass2 m_handler_pass2;
164 
165  // All relations we are interested in will be kept in this buffer
167 
168  // All members we are interested in will be kept in this buffer
170 
172  std::vector<RelationMeta> m_relations;
173 
178  std::vector<MemberMeta> m_member_meta[3];
179 
181 
182  typedef std::function<void(osmium::memory::Buffer&&)> callback_func_type;
183  callback_func_type m_callback;
184 
185  static constexpr size_t initial_buffer_size = 1024 * 1024;
186 
187  public:
188 
193  m_handler_pass2(*static_cast<TCollector*>(this)),
194  m_relations_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
195  m_members_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
196  m_relations(),
197  m_member_meta() {
198  }
199 
200  protected:
201 
202  std::vector<MemberMeta>& member_meta(const item_type type) {
203  return m_member_meta[static_cast<uint16_t>(type) - 1];
204  }
205 
206  callback_func_type callback() {
207  return m_callback;
208  }
209 
210  const std::vector<RelationMeta>& relations() const {
211  return m_relations;
212  }
213 
223  bool keep_relation(const osmium::Relation& /*relation*/) const {
224  return true;
225  }
226 
237  bool keep_member(const osmium::relations::RelationMeta& /*relation_meta*/, const osmium::RelationMember& /*member*/) const {
238  return true;
239  }
240 
248  void node_not_in_any_relation(const osmium::Node& /*node*/) {
249  }
250 
258  void way_not_in_any_relation(const osmium::Way& /*way*/) {
259  }
260 
268  void relation_not_in_any_relation(const osmium::Relation& /*relation*/) {
269  }
270 
282  void flush() {
283  }
284 
290  m_relations.erase(
291  std::remove_if(m_relations.begin(), m_relations.end(), has_all_members()),
292  m_relations.end()
293  );
294  }
295 
296  const osmium::Relation& get_relation(size_t offset) const {
297  return m_relations_buffer.get<osmium::Relation>(offset);
298  }
299 
303  const osmium::Relation& get_relation(const RelationMeta& relation_meta) const {
304  return get_relation(relation_meta.relation_offset());
305  }
306 
307  osmium::OSMObject& get_member(size_t offset) const {
308  return m_members_buffer.get<osmium::OSMObject>(offset);
309  }
310 
311  private:
312 
321  void add_relation(const osmium::Relation& relation) {
322  const size_t offset = m_relations_buffer.committed();
323  m_relations_buffer.add_item(relation);
324 
325  RelationMeta relation_meta(offset);
326 
327  int n = 0;
328  for (auto& member : m_relations_buffer.get<osmium::Relation>(offset).members()) {
329  if (static_cast<TCollector*>(this)->keep_member(relation_meta, member)) {
330  member_meta(member.type()).emplace_back(member.ref(), m_relations.size(), n);
331  relation_meta.increment_need_members();
332  } else {
333  member.ref(0); // set member id to zero to indicate we are not interested
334  }
335  ++n;
336  }
337 
338  assert(offset == m_relations_buffer.committed());
339  if (relation_meta.has_all_members()) {
340  m_relations_buffer.rollback();
341  } else {
342  m_relations_buffer.commit();
343  m_relations.push_back(std::move(relation_meta));
344 // std::cerr << "added relation id=" << relation.id() << "\n";
345  }
346  }
347 
353 /* std::cerr << "relations: " << m_relations.size() << "\n";
354  std::cerr << "node members: " << m_member_meta[0].size() << "\n";
355  std::cerr << "way members: " << m_member_meta[1].size() << "\n";
356  std::cerr << "relation members: " << m_member_meta[2].size() << "\n";*/
357  std::sort(m_member_meta[0].begin(), m_member_meta[0].end());
358  std::sort(m_member_meta[1].begin(), m_member_meta[1].end());
359  std::sort(m_member_meta[2].begin(), m_member_meta[2].end());
360  }
361 
370  auto& mmv = member_meta(object.type());
371  auto range = std::equal_range(mmv.begin(), mmv.end(), MemberMeta(object.id()));
372 
373  if (osmium::relations::count_not_removed(range.first, range.second) == 0) {
374  // nothing found
375  return false;
376  }
377 
378  {
379  members_buffer().add_item(object);
380  const size_t member_offset = members_buffer().commit();
381 
382  for (auto it = range.first; it != range.second; ++it) {
383  it->set_buffer_offset(member_offset);
384  }
385  }
386 
387  for (auto it = range.first; it != range.second; ++it) {
388  MemberMeta& member_meta = *it;
389  if (member_meta.removed()) {
390  break;
391  }
392  assert(member_meta.member_id() == object.id());
393  assert(member_meta.relation_pos() < m_relations.size());
394  RelationMeta& relation_meta = m_relations[member_meta.relation_pos()];
395 // std::cerr << " => " << member_meta.member_pos() << " < " << get_relation(relation_meta).members().size() << " (id=" << get_relation(relation_meta).id() << ")\n";
396  assert(member_meta.member_pos() < get_relation(relation_meta).members().size());
397 // std::cerr << " add way " << member_meta.member_id() << " to rel " << get_relation(relation_meta).id() << " at pos " << member_meta.member_pos() << "\n";
398  relation_meta.got_one_member();
399  if (relation_meta.has_all_members()) {
400  const size_t relation_offset = member_meta.relation_pos();
401  static_cast<TCollector*>(this)->complete_relation(relation_meta);
402  clear_member_metas(relation_meta);
403  m_relations[relation_offset] = RelationMeta();
405  }
406  }
407 
408  // Remove MemberMetas that were marked as removed.
409  mmv.erase(std::remove_if(mmv.begin(), mmv.end(), [](MemberMeta& mm) {
410  return mm.removed();
411  }), mmv.end());
412 
413  return true;
414  }
415 
416  void clear_member_metas(const osmium::relations::RelationMeta& relation_meta) {
417  const osmium::Relation& relation = get_relation(relation_meta);
418  for (const auto& member : relation.members()) {
419  if (member.ref() != 0) {
420  auto& mmv = member_meta(member.type());
421  auto range = std::equal_range(mmv.begin(), mmv.end(), MemberMeta(member.ref()));
422  assert(range.first != range.second);
423 
424  // if this is the last time this object was needed
425  // then mark it as removed
426  if (osmium::relations::count_not_removed(range.first, range.second) == 1) {
427  get_member(range.first->buffer_offset()).set_removed(true);
428  }
429 
430  for (auto it = range.first; it != range.second; ++it) {
431  if (!it->removed() && relation.id() == get_relation(it->relation_pos()).id()) {
432  it->remove();
433  break;
434  }
435  }
436  }
437  }
438  }
439 
440  public:
441 
442  uint64_t used_memory() const {
443  const uint64_t nmembers = m_member_meta[0].capacity() + m_member_meta[1].capacity() + m_member_meta[2].capacity();
444  const uint64_t members = nmembers * sizeof(MemberMeta);
445  const uint64_t relations = m_relations.capacity() * sizeof(RelationMeta);
446  const uint64_t relations_buffer_capacity = m_relations_buffer.capacity();
447  const uint64_t members_buffer_capacity = m_members_buffer.capacity();
448 
449  std::cout << " nR = m_relations.capacity() ........... = " << std::setw(12) << m_relations.capacity() << "\n";
450  std::cout << " nMN = m_member_meta[NODE].capacity() ... = " << std::setw(12) << m_member_meta[0].capacity() << "\n";
451  std::cout << " nMW = m_member_meta[WAY].capacity() .... = " << std::setw(12) << m_member_meta[1].capacity() << "\n";
452  std::cout << " nMR = m_member_meta[RELATION].capacity() = " << std::setw(12) << m_member_meta[2].capacity() << "\n";
453  std::cout << " nM = m_member_meta[*].capacity() ...... = " << std::setw(12) << nmembers << "\n";
454 
455  std::cout << " sRM = sizeof(RelationMeta) ............. = " << std::setw(12) << sizeof(RelationMeta) << "\n";
456  std::cout << " sMM = sizeof(MemberMeta) ............... = " << std::setw(12) << sizeof(MemberMeta) << "\n\n";
457 
458  std::cout << " nR * sRM ............................... = " << std::setw(12) << relations << "\n";
459  std::cout << " nM * sMM ............................... = " << std::setw(12) << members << "\n";
460  std::cout << " relations_buffer_capacity .............. = " << std::setw(12) << relations_buffer_capacity << "\n";
461  std::cout << " members_buffer_capacity ................ = " << std::setw(12) << members_buffer_capacity << "\n";
462 
463  const uint64_t total = relations + members + relations_buffer_capacity + members_buffer_capacity;
464 
465  std::cout << " total .................................. = " << std::setw(12) << total << "\n";
466  std::cout << " =======================================================\n";
467 
468  return relations_buffer_capacity + members_buffer_capacity + relations + members;
469  }
470 
474  HandlerPass2& handler(const callback_func_type& callback = nullptr) {
475  m_callback = callback;
476  return m_handler_pass2;
477  }
478 
480  return m_members_buffer;
481  }
482 
484  const auto& mmv = member_meta(type);
485  const auto range = std::equal_range(mmv.cbegin(), mmv.cend(), MemberMeta(id));
486  assert(range.first != range.second);
487  return range.first->buffer_offset();
488  }
489 
490  template <typename TIter>
491  void read_relations(TIter begin, TIter end) {
492  HandlerPass1 handler(*static_cast<TCollector*>(this));
493  osmium::apply(begin, end, handler);
495  }
496 
497  template <typename TSource>
498  void read_relations(TSource& source) {
499  read_relations(std::begin(source), std::end(source));
500  source.close();
501  }
502 
503  void moving_in_buffer(size_t old_offset, size_t new_offset) {
504  const osmium::OSMObject& object = m_members_buffer.get<osmium::OSMObject>(old_offset);
505  auto& mmv = member_meta(object.type());
506  auto range = std::equal_range(mmv.begin(), mmv.end(), MemberMeta(object.id()));
507  for (auto it = range.first; it != range.second; ++it) {
508  assert(it->buffer_offset() == old_offset);
509  it->set_buffer_offset(new_offset);
510  }
511  }
512 
521  if (m_count_complete > 10000) { // XXX
522 // const size_t size_before = m_members_buffer.committed();
523  m_members_buffer.purge_removed(this);
524 /*
525  const size_t size_after = m_members_buffer.committed();
526  double percent = static_cast<double>(size_before - size_after);
527  percent /= size_before;
528  percent *= 100;
529  std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
530 */
531  m_count_complete = 0;
532  }
533  }
534 
543  std::vector<const osmium::Relation*> get_incomplete_relations() const {
544  std::vector<const osmium::Relation*> relations;
545  for (const auto& relation_meta : m_relations) {
546  if (!relation_meta.has_all_members()) {
547  relations.push_back(&get_relation(relation_meta));
548  }
549  }
550  return relations;
551  }
552 
553  }; // class Collector
554 
555  } // namespace relations
556 
557 } // namespace osmium
558 
559 #endif // OSMIUM_RELATIONS_COLLECTOR_HPP
std::vector< const osmium::Relation * > get_incomplete_relations() const
Definition: collector.hpp:543
void relation(const osmium::Relation &relation)
Definition: collector.hpp:147
void clear_member_metas(const osmium::relations::RelationMeta &relation_meta)
Definition: collector.hpp:416
callback_func_type m_callback
Definition: collector.hpp:183
osmium::memory::Buffer & members_buffer()
Definition: collector.hpp:479
type
Definition: entity_bits.hpp:60
RelationMemberList & members()
Definition: relation.hpp:177
void way(const osmium::Way &way)
Definition: collector.hpp:139
item_type
Definition: item_type.hpp:43
void clean_assembled_relations()
Definition: collector.hpp:289
void moving_in_buffer(size_t old_offset, size_t new_offset)
Definition: collector.hpp:503
Definition: relation.hpp:165
static constexpr size_t initial_buffer_size
Definition: collector.hpp:185
Definition: handler.hpp:45
bool keep_member(const osmium::relations::RelationMeta &, const osmium::RelationMember &) const
Definition: collector.hpp:237
osmium::memory::Buffer m_members_buffer
Definition: collector.hpp:169
const std::vector< RelationMeta > & relations() const
Definition: collector.hpp:210
void read_relations(TSource &source)
Definition: collector.hpp:498
size_t get_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:483
void relation_not_in_any_relation(const osmium::Relation &)
Definition: collector.hpp:268
Definition: way.hpp:65
std::vector< MemberMeta > & member_meta(const item_type type)
Definition: collector.hpp:202
bool find_and_add_object(const osmium::OSMObject &object)
Definition: collector.hpp:369
Definition: collector.hpp:121
HandlerPass2(TCollector &collector) noexcept
Definition: collector.hpp:127
std::vector< RelationMeta > m_relations
Vector with all relations we are interested in.
Definition: collector.hpp:172
void sort_member_meta()
Definition: collector.hpp:352
void apply(TIterator it, TIterator end, THandlers &...handlers)
Definition: visitor.hpp:234
iterator end()
Definition: object.hpp:338
void node_not_in_any_relation(const osmium::Node &)
Definition: collector.hpp:248
Definition: relation.hpp:54
TCollector & m_collector
Definition: collector.hpp:100
int64_t object_id_type
Type for OSM object (node, way, or relation) IDs.
Definition: types.hpp:45
Namespace for everything in the Osmium library.
Definition: assembler.hpp:59
T & add_item(const T &item)
Definition: buffer.hpp:457
Definition: collector.hpp:93
const osmium::Relation & get_relation(size_t offset) const
Definition: collector.hpp:296
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:690
HandlerPass2 m_handler_pass2
Definition: collector.hpp:163
Collector()
Definition: collector.hpp:192
std::function< void(osmium::memory::Buffer &&)> callback_func_type
Definition: collector.hpp:182
osmium::OSMObject & get_member(size_t offset) const
Definition: collector.hpp:307
size_t capacity() const noexcept
Definition: buffer.hpp:233
void relation(const osmium::Relation &relation)
Definition: collector.hpp:108
osmium::io::InputIterator< osmium::io::Reader > end(osmium::io::Reader &)
Definition: reader_iterator.hpp:45
Definition: collector.hpp:98
void flush()
Definition: collector.hpp:155
osmium::memory::Buffer m_relations_buffer
Definition: collector.hpp:166
object_id_type id() const noexcept
Get ID of this object.
Definition: object.hpp:110
uint64_t used_memory() const
Definition: collector.hpp:442
size_t committed() const noexcept
Definition: buffer.hpp:241
callback_func_type callback()
Definition: collector.hpp:206
Definition: buffer.hpp:97
T & get(const size_t offset) const
Definition: buffer.hpp:379
void add_relation(const osmium::Relation &relation)
Definition: collector.hpp:321
HandlerPass2 & handler(const callback_func_type &callback=nullptr)
Definition: collector.hpp:474
void flush()
Definition: collector.hpp:282
Definition: node.hpp:47
void way_not_in_any_relation(const osmium::Way &)
Definition: collector.hpp:258
int m_count_complete
Definition: collector.hpp:180
void node(const osmium::Node &node)
Definition: collector.hpp:131
void possibly_purge_removed_members()
Definition: collector.hpp:519
bool keep_relation(const osmium::Relation &) const
Definition: collector.hpp:223
TCollector & m_collector
Definition: collector.hpp:123
void read_relations(TIter begin, TIter end)
Definition: collector.hpp:491
osmium::io::InputIterator< osmium::io::Reader > begin(osmium::io::Reader &reader)
Definition: reader_iterator.hpp:41
HandlerPass1(TCollector &collector) noexcept
Definition: collector.hpp:104
void rollback()
Definition: buffer.hpp:349
const osmium::Relation & get_relation(const RelationMeta &relation_meta) const
Definition: collector.hpp:303
Definition: object.hpp:58
size_t commit()
Definition: buffer.hpp:335
std::vector< MemberMeta > m_member_meta[3]
Definition: collector.hpp:178
size_type size() const noexcept
Definition: relation.hpp:158
iterator begin()
Definition: object.hpp:334