Libosmium  2.4.1
Fast and flexible C++ library for working with OpenStreetMap data
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
memory_mapping.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_UTIL_MEMORY_MAPPING_HPP
2 #define OSMIUM_UTIL_MEMORY_MAPPING_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <cassert>
37 #include <cerrno>
38 #include <stdexcept>
39 #include <system_error>
40 
41 #include <osmium/util/file.hpp>
42 
43 #ifndef _WIN32
44 # include <sys/mman.h>
45 #else
46 # include <fcntl.h>
47 # include <io.h>
48 # include <windows.h>
49 # include <sys/types.h>
50 #endif
51 
52 namespace osmium {
53 
54  namespace util {
55 
93  class MemoryMapping {
94 
95 public:
96  enum class mapping_mode {
97  readonly = 0,
98  write_private = 1,
99  write_shared = 2
100  };
101 
102 private:
103 
105  size_t m_size;
106 
108  off_t m_offset;
109 
111  int m_fd;
112 
115 
116 #ifdef _WIN32
117  HANDLE m_handle;
118 #endif
119 
121  void* m_addr;
122 
123  bool is_valid() const noexcept;
124 
125  void make_invalid() noexcept;
126 
127 #ifdef _WIN32
128  typedef DWORD flag_type;
129 #else
130  typedef int flag_type;
131 #endif
132 
133  flag_type get_protection() const noexcept;
134 
135  flag_type get_flags() const noexcept;
136 
137  // A zero-sized mapping is not allowed by the operating system.
138  // So if the user asks for a mapping of size 0, we map a full
139  // page instead. This way we don't have a special case in the rest
140  // of the code.
141  static size_t initial_size(size_t size) {
142  if (size == 0) {
144  }
145  return size;
146  }
147 
148 #ifdef _WIN32
149  HANDLE get_handle() const noexcept;
150  HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept;
151  void* osmium::util::MemoryMapping::map_view_of_file() const noexcept;
152 #endif
153 
154  int resize_fd(int fd) {
155  // Anonymous mapping doesn't need resizing.
156  if (fd == -1) {
157  return -1;
158  }
159 
160  // Make sure the file backing this mapping is large enough.
161  if (osmium::util::file_size(fd) < m_size + m_offset) {
162  osmium::util::resize_file(fd, m_size + m_offset);
163  }
164  return fd;
165  }
166 
167  public:
168 
184  MemoryMapping(size_t size, mapping_mode mode, int fd=-1, off_t offset=0);
185 
187  MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0) :
188  MemoryMapping(size, writable ? mapping_mode::write_shared : mapping_mode::readonly, fd, offset) {
189  }
190 
192  MemoryMapping(const MemoryMapping&) = delete;
193 
195  MemoryMapping& operator=(const MemoryMapping&) = delete;
196 
201  MemoryMapping(MemoryMapping&& other);
202 
207 
212  ~MemoryMapping() noexcept {
213  try {
214  unmap();
215  } catch (std::system_error&) {
216  // ignore
217  }
218  }
219 
226  void unmap();
227 
238  void resize(size_t new_size);
239 
244  operator bool() const noexcept {
245  return is_valid();
246  }
247 
253  size_t size() const noexcept {
254  return m_size;
255  }
256 
262  int fd() const noexcept {
263  return m_fd;
264  }
265 
269  bool writable() const noexcept {
270  return m_mapping_mode != mapping_mode::readonly;
271  }
272 
278  template <typename T = void>
279  T* get_addr() const {
280  if (is_valid()) {
281  return reinterpret_cast<T*>(m_addr);
282  }
283  throw std::runtime_error("invalid memory mapping");
284  }
285 
286  }; // class MemoryMapping
287 
299 
300  public:
301 
303  MemoryMapping(size, mapping_mode::write_private) {
304  }
305 
306 #ifndef __linux__
307 
311  void resize(size_t) = delete;
312 #endif
313 
314  }; // class AnonymousMemoryMapping
315 
325  template <typename T>
327 
329 
330  public:
331 
339  m_mapping(sizeof(T) * size, MemoryMapping::mapping_mode::write_private) {
340  }
341 
352  TypedMemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset = 0) :
353  m_mapping(sizeof(T) * size, mode, fd, sizeof(T) * offset) {
354  }
355 
357  TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset = 0) :
358  m_mapping(sizeof(T) * size, writable ? MemoryMapping::mapping_mode::write_shared : MemoryMapping::mapping_mode::readonly, fd, sizeof(T) * offset) {
359  }
360 
362  TypedMemoryMapping(const TypedMemoryMapping&) = delete;
363 
366 
371  TypedMemoryMapping(TypedMemoryMapping&& other) = default;
372 
376  TypedMemoryMapping& operator=(TypedMemoryMapping&& other) = default;
377 
382  ~TypedMemoryMapping() = default;
383 
390  void unmap() {
391  m_mapping.unmap();
392  }
393 
404  void resize(size_t new_size) {
405  m_mapping.resize(sizeof(T) * new_size);
406  }
407 
412  operator bool() const noexcept {
413  return !!m_mapping;
414  }
415 
421  size_t size() const noexcept {
422  assert(m_mapping.size() % sizeof(T) == 0);
423  return m_mapping.size() / sizeof(T);
424  }
425 
431  int fd() const noexcept {
432  return m_mapping.fd();
433  }
434 
438  bool writable() const noexcept {
439  return m_mapping.writable();
440  }
441 
447  T* begin() {
448  return m_mapping.get_addr<T>();
449  }
450 
456  T* end() {
457  return m_mapping.get_addr<T>() + size();
458  }
459 
460  const T* cbegin() const {
461  return m_mapping.get_addr<T>();
462  }
463 
464  const T* cend() const {
465  return m_mapping.get_addr<T>() + size();
466  }
467 
468  const T* begin() const {
469  return m_mapping.get_addr<T>();
470  }
471 
472  const T* end() const {
473  return m_mapping.get_addr<T>() + size();
474  }
475 
476  }; // class TypedMemoryMapping
477 
478  template <typename T>
480 
481  public:
482 
484  TypedMemoryMapping<T>(size) {
485  }
486 
487 #ifndef __linux__
488 
492  void resize(size_t) = delete;
493 #endif
494 
495  }; // class AnonymousTypedMemoryMapping
496 
497  } // namespace util
498 
499 } // namespace osmium
500 
501 #ifndef _WIN32
502 
503 // =========== Unix implementation =============
504 
505 // MAP_FAILED is often a macro containing an old style cast
506 #pragma GCC diagnostic push
507 #pragma GCC diagnostic ignored "-Wold-style-cast"
508 
509 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
510  return m_addr != MAP_FAILED;
511 }
512 
514  m_addr = MAP_FAILED;
515 }
516 
517 #pragma GCC diagnostic pop
518 
519 // for BSD systems
520 #ifndef MAP_ANONYMOUS
521 # define MAP_ANONYMOUS MAP_ANON
522 #endif
523 
524 inline int osmium::util::MemoryMapping::get_protection() const noexcept {
525  if (m_mapping_mode == mapping_mode::readonly) {
526  return PROT_READ;
527  }
528  return PROT_READ | PROT_WRITE;
529 }
530 
531 inline int osmium::util::MemoryMapping::get_flags() const noexcept {
532  if (m_fd == -1) {
533  return MAP_PRIVATE | MAP_ANONYMOUS;
534  }
535  if (m_mapping_mode == mapping_mode::write_shared) {
536  return MAP_SHARED;
537  }
538  return MAP_PRIVATE;
539 }
540 
541 inline osmium::util::MemoryMapping::MemoryMapping(size_t size, mapping_mode mode, int fd, off_t offset) :
542  m_size(initial_size(size)),
543  m_offset(offset),
544  m_fd(resize_fd(fd)),
545  m_mapping_mode(mode),
546  m_addr(::mmap(nullptr, m_size, get_protection(), get_flags(), m_fd, m_offset)) {
547  assert(!(fd == -1 && mode == mapping_mode::readonly));
548  if (!is_valid()) {
549  throw std::system_error(errno, std::system_category(), "mmap failed");
550  }
551 }
552 
554  m_size(other.m_size),
555  m_offset(other.m_offset),
556  m_fd(other.m_fd),
557  m_mapping_mode(other.m_mapping_mode),
558  m_addr(other.m_addr) {
559  other.make_invalid();
560 }
561 
563  unmap();
564  m_size = other.m_size;
565  m_offset = other.m_offset;
566  m_fd = other.m_fd;
567  m_mapping_mode = other.m_mapping_mode;
568  m_addr = other.m_addr;
569  other.make_invalid();
570  return *this;
571 }
572 
574  if (is_valid()) {
575  if (::munmap(m_addr, m_size) != 0) {
576  throw std::system_error(errno, std::system_category(), "munmap failed");
577  }
578  make_invalid();
579  }
580 }
581 
582 inline void osmium::util::MemoryMapping::resize(size_t new_size) {
583  assert(new_size > 0 && "can not resize to zero size");
584  if (m_fd == -1) { // anonymous mapping
585 #ifdef __linux__
586  m_addr = ::mremap(m_addr, m_size, new_size, MREMAP_MAYMOVE);
587  if (!is_valid()) {
588  throw std::system_error(errno, std::system_category(), "mremap failed");
589  }
590  m_size = new_size;
591 #else
592  assert(false && "can't resize anonymous mappings on non-linux systems");
593 #endif
594  } else { // file-based mapping
595  unmap();
596  m_size = new_size;
597  resize_fd(m_fd);
598  m_addr = ::mmap(nullptr, new_size, get_protection(), get_flags(), m_fd, m_offset);
599  if (!is_valid()) {
600  throw std::system_error(errno, std::system_category(), "mmap (remap) failed");
601  }
602  }
603 }
604 
605 #else
606 
607 // =========== Windows implementation =============
608 
609 /* References:
610  * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx
611  * CloseHandle: http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx
612  * MapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx
613  * UnmapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx
614  */
615 
616 namespace osmium {
617 
618  namespace util {
619 
620  inline DWORD dword_hi(uint64_t x) {
621  return static_cast<DWORD>(x >> 32);
622  }
623 
624  inline DWORD dword_lo(uint64_t x) {
625  return static_cast<DWORD>(x & 0xffffffff);
626  }
627 
628  } // namespace util
629 
630 } // namespace osmium
631 
632 inline DWORD osmium::util::MemoryMapping::get_protection() const noexcept {
633  switch (m_mapping_mode) {
634  case mapping_mode::readonly:
635  return PAGE_READONLY;
636  case mapping_mode::write_private:
637  return PAGE_WRITECOPY;
638  case mapping_mode::write_shared:
639  return PAGE_READWRITE;
640  }
641 }
642 
643 inline DWORD osmium::util::MemoryMapping::get_flags() const noexcept {
644  switch (m_mapping_mode) {
645  case mapping_mode::readonly:
646  return FILE_MAP_READ;
647  case mapping_mode::write_private:
648  return FILE_MAP_COPY;
649  case mapping_mode::write_shared:
650  return FILE_MAP_WRITE;
651  }
652 }
653 
654 inline HANDLE osmium::util::MemoryMapping::get_handle() const noexcept {
655  if (m_fd == -1) {
656  return INVALID_HANDLE_VALUE;
657  }
658  return reinterpret_cast<HANDLE>(_get_osfhandle(m_fd));
659 }
660 
661 inline HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept {
662  if (m_fd != -1) {
663  _setmode(m_fd, _O_BINARY);
664  }
665  return CreateFileMapping(get_handle(), nullptr, get_protection(), osmium::util::dword_hi(static_cast<uint64_t>(m_size) + m_offset), osmium::util::dword_lo(static_cast<uint64_t>(m_size) + m_offset), nullptr);
666 }
667 
668 inline void* osmium::util::MemoryMapping::map_view_of_file() const noexcept {
669  return MapViewOfFile(m_handle, get_flags(), osmium::util::dword_hi(m_offset), osmium::util::dword_lo(m_offset), m_size);
670 }
671 
672 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
673  return m_addr != nullptr;
674 }
675 
676 inline void osmium::util::MemoryMapping::make_invalid() noexcept {
677  m_addr = nullptr;
678 }
679 
680 inline osmium::util::MemoryMapping::MemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset) :
681  m_size(initial_size(size)),
682  m_offset(offset),
683  m_fd(resize_fd(fd)),
684  m_mapping_mode(mode),
685  m_handle(create_file_mapping()),
686  m_addr(nullptr) {
687 
688  if (!m_handle) {
689  throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed");
690  }
691 
692  m_addr = map_view_of_file();
693  if (!is_valid()) {
694  throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed");
695  }
696 }
697 
698 inline osmium::util::MemoryMapping::MemoryMapping(MemoryMapping&& other) :
699  m_size(other.m_size),
700  m_offset(other.m_offset),
701  m_fd(other.m_fd),
702  m_mapping_mode(other.m_mapping_mode),
703  m_handle(std::move(other.m_handle)),
704  m_addr(other.m_addr) {
705  other.make_invalid();
706  other.m_handle = nullptr;
707 }
708 
710  unmap();
711  m_size = other.m_size;
712  m_offset = other.m_offset;
713  m_fd = other.m_fd;
714  m_mapping_mode = other.m_mapping_mode;
715  m_handle = std::move(other.m_handle);
716  m_addr = other.m_addr;
717  other.make_invalid();
718  other.m_handle = nullptr;
719  return *this;
720 }
721 
723  if (is_valid()) {
724  if (! UnmapViewOfFile(m_addr)) {
725  throw std::system_error(GetLastError(), std::system_category(), "UnmapViewOfFile failed");
726  }
727  make_invalid();
728  }
729 
730  if (m_handle) {
731  if (! CloseHandle(m_handle)) {
732  throw std::system_error(GetLastError(), std::system_category(), "CloseHandle failed");
733  }
734  m_handle = nullptr;
735  }
736 }
737 
738 inline void osmium::util::MemoryMapping::resize(size_t new_size) {
739  unmap();
740 
741  m_size = new_size;
742  resize_fd(m_fd);
743 
744  m_handle = create_file_mapping();
745  if (!m_handle) {
746  throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed");
747  }
748 
749  m_addr = map_view_of_file();
750  if (!is_valid()) {
751  throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed");
752  }
753 }
754 
755 #endif
756 
757 #endif // OSMIUM_UTIL_MEMORY_MAPPING_HPP
~MemoryMapping() noexcept
Definition: memory_mapping.hpp:212
const T * end() const
Definition: memory_mapping.hpp:472
bool is_valid() const noexcept
Definition: memory_mapping.hpp:509
flag_type get_protection() const noexcept
Definition: memory_mapping.hpp:524
MemoryMapping m_mapping
Definition: memory_mapping.hpp:328
MemoryMapping(size_t size, mapping_mode mode, int fd=-1, off_t offset=0)
Definition: memory_mapping.hpp:541
size_t file_size(int fd)
Definition: file.hpp:67
flag_type get_flags() const noexcept
Definition: memory_mapping.hpp:531
int fd() const noexcept
Definition: memory_mapping.hpp:262
TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset=0)
DEPRECATED: For backwards compatibility.
Definition: memory_mapping.hpp:357
static size_t initial_size(size_t size)
Definition: memory_mapping.hpp:141
void unmap()
Definition: memory_mapping.hpp:573
Definition: memory_mapping.hpp:93
Definition: reader_iterator.hpp:39
mapping_mode
Definition: memory_mapping.hpp:96
void resize(size_t new_size)
Definition: memory_mapping.hpp:404
int flag_type
Definition: memory_mapping.hpp:130
int resize_fd(int fd)
Definition: memory_mapping.hpp:154
T * end()
Definition: memory_mapping.hpp:456
size_t get_pagesize()
Definition: file.hpp:103
void * m_addr
The address where the memory is mapped.
Definition: memory_mapping.hpp:121
const T * cbegin() const
Definition: memory_mapping.hpp:460
off_t m_offset
Offset into the file.
Definition: memory_mapping.hpp:108
#define MAP_ANONYMOUS
Definition: memory_mapping.hpp:521
int m_fd
File handle we got the mapping from.
Definition: memory_mapping.hpp:111
size_t size() const noexcept
Definition: memory_mapping.hpp:253
mapping_mode m_mapping_mode
Mapping mode.
Definition: memory_mapping.hpp:114
Namespace for everything in the Osmium library.
Definition: assembler.hpp:55
AnonymousMemoryMapping(size_t size)
Definition: memory_mapping.hpp:302
Definition: memory_mapping.hpp:479
TypedMemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset=0)
Definition: memory_mapping.hpp:352
size_t size() const noexcept
Definition: memory_mapping.hpp:421
T * begin()
Definition: memory_mapping.hpp:447
Definition: memory_mapping.hpp:298
void make_invalid() noexcept
Definition: memory_mapping.hpp:513
MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0)
DEPRECATED: For backwards compatibility.
Definition: memory_mapping.hpp:187
AnonymousTypedMemoryMapping(size_t size)
Definition: memory_mapping.hpp:483
int fd() const noexcept
Definition: memory_mapping.hpp:431
void resize(size_t new_size)
Definition: memory_mapping.hpp:582
MemoryMapping & operator=(const MemoryMapping &)=delete
You can not copy a MemoryMapping.
Definition: memory_mapping.hpp:326
TypedMemoryMapping & operator=(const TypedMemoryMapping &)=delete
You can not copy a MemoryMapping.
void unmap()
Definition: memory_mapping.hpp:390
bool writable() const noexcept
Definition: memory_mapping.hpp:269
void resize_file(int fd, size_t new_size)
Definition: file.hpp:94
const T * begin() const
Definition: memory_mapping.hpp:468
size_t m_size
The size of the mapping.
Definition: memory_mapping.hpp:105
const T * cend() const
Definition: memory_mapping.hpp:464
TypedMemoryMapping(size_t size)
Definition: memory_mapping.hpp:338
T * get_addr() const
Definition: memory_mapping.hpp:279
bool writable() const noexcept
Definition: memory_mapping.hpp:438