Libosmium  2.10.3
Fast and flexible C++ library for working with OpenStreetMap data
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
memory_mapping.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_UTIL_MEMORY_MAPPING_HPP
2 #define OSMIUM_UTIL_MEMORY_MAPPING_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <cassert>
37 #include <cerrno>
38 #include <cstddef>
39 #include <stdexcept>
40 #include <system_error>
41 
43 #include <osmium/util/file.hpp>
44 
45 #ifndef _WIN32
46 # include <sys/mman.h>
47 #else
48 # include <fcntl.h>
49 # include <io.h>
50 # include <windows.h>
51 # include <sys/types.h>
52 #endif
53 
54 namespace osmium {
55 
56  namespace util {
57 
95  class MemoryMapping {
96 
97 public:
98  enum class mapping_mode {
99  readonly = 0,
100  write_private = 1,
101  write_shared = 2
102  };
103 
104 private:
105 
107  size_t m_size;
108 
110  off_t m_offset;
111 
113  int m_fd;
114 
117 
118 #ifdef _WIN32
119  HANDLE m_handle;
120 #endif
121 
123  void* m_addr;
124 
125  bool is_valid() const noexcept;
126 
127  void make_invalid() noexcept;
128 
129 #ifdef _WIN32
130  using flag_type = DWORD;
131 #else
132  using flag_type = int;
133 #endif
134 
135  flag_type get_protection() const noexcept;
136 
137  flag_type get_flags() const noexcept;
138 
139  static size_t check_size(size_t size) {
140  if (size == 0) {
141  throw std::runtime_error("Zero-sized mapping is not allowed.");
142  }
143  return size;
144  }
145 
146 #ifdef _WIN32
147  HANDLE get_handle() const noexcept;
148  HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept;
149  void* osmium::util::MemoryMapping::map_view_of_file() const noexcept;
150 #endif
151 
152  int resize_fd(int fd) {
153  // Anonymous mapping doesn't need resizing.
154  if (fd == -1) {
155  return -1;
156  }
157 
158  // Make sure the file backing this mapping is large enough.
159  if (osmium::util::file_size(fd) < m_size + m_offset) {
160  osmium::util::resize_file(fd, m_size + m_offset);
161  }
162  return fd;
163  }
164 
165  public:
166 
183  MemoryMapping(size_t size, mapping_mode mode, int fd=-1, off_t offset=0);
184 
190  OSMIUM_DEPRECATED MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0) :
191  MemoryMapping(size, writable ? mapping_mode::write_shared : mapping_mode::readonly, fd, offset) {
192  }
193 
195  MemoryMapping(const MemoryMapping&) = delete;
196 
198  MemoryMapping& operator=(const MemoryMapping&) = delete;
199 
204  MemoryMapping(MemoryMapping&& other);
205 
210 
215  ~MemoryMapping() noexcept {
216  try {
217  unmap();
218  } catch (const std::system_error&) {
219  // Ignore any exceptions because destructor must not throw.
220  }
221  }
222 
229  void unmap();
230 
242  void resize(size_t new_size);
243 
248  explicit operator bool() const noexcept {
249  return is_valid();
250  }
251 
257  size_t size() const noexcept {
258  return m_size;
259  }
260 
266  int fd() const noexcept {
267  return m_fd;
268  }
269 
273  bool writable() const noexcept {
274  return m_mapping_mode != mapping_mode::readonly;
275  }
276 
282  template <typename T = void>
283  T* get_addr() const {
284  if (is_valid()) {
285  return reinterpret_cast<T*>(m_addr);
286  }
287  throw std::runtime_error("invalid memory mapping");
288  }
289 
290  }; // class MemoryMapping
291 
303 
304  public:
305 
306  explicit AnonymousMemoryMapping(size_t size) :
307  MemoryMapping(size, mapping_mode::write_private) {
308  }
309 
310 #ifndef __linux__
311 
315  void resize(size_t) = delete;
316 #endif
317 
318  }; // class AnonymousMemoryMapping
319 
329  template <typename T>
331 
333 
334  public:
335 
342  explicit TypedMemoryMapping(size_t size) :
343  m_mapping(sizeof(T) * size, MemoryMapping::mapping_mode::write_private) {
344  }
345 
356  TypedMemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset = 0) :
357  m_mapping(sizeof(T) * size, mode, fd, sizeof(T) * offset) {
358  }
359 
365  OSMIUM_DEPRECATED TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset = 0) :
366  m_mapping(sizeof(T) * size, writable ? MemoryMapping::mapping_mode::write_shared : MemoryMapping::mapping_mode::readonly, fd, sizeof(T) * offset) {
367  }
368 
370  TypedMemoryMapping(const TypedMemoryMapping&) = delete;
371 
374 
379  TypedMemoryMapping(TypedMemoryMapping&& other) = default;
380 
384  TypedMemoryMapping& operator=(TypedMemoryMapping&& other) = default;
385 
390  ~TypedMemoryMapping() noexcept = default;
391 
398  void unmap() {
399  m_mapping.unmap();
400  }
401 
412  void resize(size_t new_size) {
413  m_mapping.resize(sizeof(T) * new_size);
414  }
415 
420  explicit operator bool() const noexcept {
421  return !!m_mapping;
422  }
423 
429  size_t size() const noexcept {
430  assert(m_mapping.size() % sizeof(T) == 0);
431  return m_mapping.size() / sizeof(T);
432  }
433 
439  int fd() const noexcept {
440  return m_mapping.fd();
441  }
442 
446  bool writable() const noexcept {
447  return m_mapping.writable();
448  }
449 
455  T* begin() {
456  return m_mapping.get_addr<T>();
457  }
458 
464  T* end() {
465  return m_mapping.get_addr<T>() + size();
466  }
467 
468  const T* cbegin() const {
469  return m_mapping.get_addr<T>();
470  }
471 
472  const T* cend() const {
473  return m_mapping.get_addr<T>() + size();
474  }
475 
476  const T* begin() const {
477  return m_mapping.get_addr<T>();
478  }
479 
480  const T* end() const {
481  return m_mapping.get_addr<T>() + size();
482  }
483 
484  }; // class TypedMemoryMapping
485 
486  template <typename T>
488 
489  public:
490 
492  TypedMemoryMapping<T>(size) {
493  }
494 
495 #ifndef __linux__
496 
500  void resize(size_t) = delete;
501 #endif
502 
503  }; // class AnonymousTypedMemoryMapping
504 
505  } // namespace util
506 
507 } // namespace osmium
508 
509 #ifndef _WIN32
510 
511 // =========== Unix implementation =============
512 
513 // MAP_FAILED is often a macro containing an old style cast
514 #pragma GCC diagnostic push
515 #pragma GCC diagnostic ignored "-Wold-style-cast"
516 
517 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
518  return m_addr != MAP_FAILED;
519 }
520 
522  m_addr = MAP_FAILED;
523 }
524 
525 #pragma GCC diagnostic pop
526 
527 // for BSD systems
528 #ifndef MAP_ANONYMOUS
529 # define MAP_ANONYMOUS MAP_ANON
530 #endif
531 
532 inline int osmium::util::MemoryMapping::get_protection() const noexcept {
533  if (m_mapping_mode == mapping_mode::readonly) {
534  return PROT_READ;
535  }
536  return PROT_READ | PROT_WRITE;
537 }
538 
539 inline int osmium::util::MemoryMapping::get_flags() const noexcept {
540  if (m_fd == -1) {
541  return MAP_PRIVATE | MAP_ANONYMOUS;
542  }
543  if (m_mapping_mode == mapping_mode::write_shared) {
544  return MAP_SHARED;
545  }
546  return MAP_PRIVATE;
547 }
548 
549 inline osmium::util::MemoryMapping::MemoryMapping(size_t size, mapping_mode mode, int fd, off_t offset) :
550  m_size(check_size(size)),
551  m_offset(offset),
552  m_fd(resize_fd(fd)),
553  m_mapping_mode(mode),
554  m_addr(::mmap(nullptr, m_size, get_protection(), get_flags(), m_fd, m_offset)) {
555  assert(!(fd == -1 && mode == mapping_mode::readonly));
556  if (!is_valid()) {
557  throw std::system_error(errno, std::system_category(), "mmap failed");
558  }
559 }
560 
562  m_size(other.m_size),
563  m_offset(other.m_offset),
564  m_fd(other.m_fd),
565  m_mapping_mode(other.m_mapping_mode),
566  m_addr(other.m_addr) {
567  other.make_invalid();
568 }
569 
571  unmap();
572  m_size = other.m_size;
573  m_offset = other.m_offset;
574  m_fd = other.m_fd;
575  m_mapping_mode = other.m_mapping_mode;
576  m_addr = other.m_addr;
577  other.make_invalid();
578  return *this;
579 }
580 
582  if (is_valid()) {
583  if (::munmap(m_addr, m_size) != 0) {
584  throw std::system_error(errno, std::system_category(), "munmap failed");
585  }
586  make_invalid();
587  }
588 }
589 
590 inline void osmium::util::MemoryMapping::resize(size_t new_size) {
591  assert(new_size > 0 && "can not resize to zero size");
592  if (m_fd == -1) { // anonymous mapping
593 #ifdef __linux__
594  m_addr = ::mremap(m_addr, m_size, new_size, MREMAP_MAYMOVE);
595  if (!is_valid()) {
596  throw std::system_error(errno, std::system_category(), "mremap failed");
597  }
598  m_size = new_size;
599 #else
600  assert(false && "can't resize anonymous mappings on non-linux systems");
601 #endif
602  } else { // file-based mapping
603  unmap();
604  m_size = new_size;
605  resize_fd(m_fd);
606  m_addr = ::mmap(nullptr, new_size, get_protection(), get_flags(), m_fd, m_offset);
607  if (!is_valid()) {
608  throw std::system_error(errno, std::system_category(), "mmap (remap) failed");
609  }
610  }
611 }
612 
613 #else
614 
615 // =========== Windows implementation =============
616 
617 /* References:
618  * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx
619  * CloseHandle: http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx
620  * MapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx
621  * UnmapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx
622  */
623 
624 namespace osmium {
625 
626  namespace util {
627 
628  inline DWORD dword_hi(uint64_t x) {
629  return static_cast<DWORD>(x >> 32);
630  }
631 
632  inline DWORD dword_lo(uint64_t x) {
633  return static_cast<DWORD>(x & 0xffffffff);
634  }
635 
636  } // namespace util
637 
638 } // namespace osmium
639 
640 inline DWORD osmium::util::MemoryMapping::get_protection() const noexcept {
641  switch (m_mapping_mode) {
642  case mapping_mode::readonly:
643  return PAGE_READONLY;
644  case mapping_mode::write_private:
645  return PAGE_WRITECOPY;
646  case mapping_mode::write_shared:
647  return PAGE_READWRITE;
648  }
649 }
650 
651 inline DWORD osmium::util::MemoryMapping::get_flags() const noexcept {
652  switch (m_mapping_mode) {
653  case mapping_mode::readonly:
654  return FILE_MAP_READ;
655  case mapping_mode::write_private:
656  return FILE_MAP_COPY;
657  case mapping_mode::write_shared:
658  return FILE_MAP_WRITE;
659  }
660 }
661 
662 inline HANDLE osmium::util::MemoryMapping::get_handle() const noexcept {
663  if (m_fd == -1) {
664  return INVALID_HANDLE_VALUE;
665  }
666  return reinterpret_cast<HANDLE>(_get_osfhandle(m_fd));
667 }
668 
669 inline HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept {
670  if (m_fd != -1) {
671  _setmode(m_fd, _O_BINARY);
672  }
673  return CreateFileMapping(get_handle(), nullptr, get_protection(), osmium::util::dword_hi(static_cast<uint64_t>(m_size) + m_offset), osmium::util::dword_lo(static_cast<uint64_t>(m_size) + m_offset), nullptr);
674 }
675 
676 inline void* osmium::util::MemoryMapping::map_view_of_file() const noexcept {
677  return MapViewOfFile(m_handle, get_flags(), osmium::util::dword_hi(m_offset), osmium::util::dword_lo(m_offset), m_size);
678 }
679 
680 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
681  return m_addr != nullptr;
682 }
683 
684 inline void osmium::util::MemoryMapping::make_invalid() noexcept {
685  m_addr = nullptr;
686 }
687 
688 inline osmium::util::MemoryMapping::MemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset) :
689  m_size(check_size(size)),
690  m_offset(offset),
691  m_fd(resize_fd(fd)),
692  m_mapping_mode(mode),
693  m_handle(create_file_mapping()),
694  m_addr(nullptr) {
695 
696  if (!m_handle) {
697  throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed");
698  }
699 
700  m_addr = map_view_of_file();
701  if (!is_valid()) {
702  throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed");
703  }
704 }
705 
706 inline osmium::util::MemoryMapping::MemoryMapping(MemoryMapping&& other) :
707  m_size(other.m_size),
708  m_offset(other.m_offset),
709  m_fd(other.m_fd),
710  m_mapping_mode(other.m_mapping_mode),
711  m_handle(std::move(other.m_handle)),
712  m_addr(other.m_addr) {
713  other.make_invalid();
714  other.m_handle = nullptr;
715 }
716 
718  unmap();
719  m_size = other.m_size;
720  m_offset = other.m_offset;
721  m_fd = other.m_fd;
722  m_mapping_mode = other.m_mapping_mode;
723  m_handle = std::move(other.m_handle);
724  m_addr = other.m_addr;
725  other.make_invalid();
726  other.m_handle = nullptr;
727  return *this;
728 }
729 
731  if (is_valid()) {
732  if (! UnmapViewOfFile(m_addr)) {
733  throw std::system_error(GetLastError(), std::system_category(), "UnmapViewOfFile failed");
734  }
735  make_invalid();
736  }
737 
738  if (m_handle) {
739  if (! CloseHandle(m_handle)) {
740  throw std::system_error(GetLastError(), std::system_category(), "CloseHandle failed");
741  }
742  m_handle = nullptr;
743  }
744 }
745 
746 inline void osmium::util::MemoryMapping::resize(size_t new_size) {
747  unmap();
748 
749  m_size = new_size;
750  resize_fd(m_fd);
751 
752  m_handle = create_file_mapping();
753  if (!m_handle) {
754  throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed");
755  }
756 
757  m_addr = map_view_of_file();
758  if (!is_valid()) {
759  throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed");
760  }
761 }
762 
763 #endif
764 
765 #endif // OSMIUM_UTIL_MEMORY_MAPPING_HPP
~MemoryMapping() noexcept
Definition: memory_mapping.hpp:215
const T * end() const
Definition: memory_mapping.hpp:480
bool is_valid() const noexcept
Definition: memory_mapping.hpp:517
#define OSMIUM_DEPRECATED
Definition: compatibility.hpp:50
flag_type get_protection() const noexcept
Definition: memory_mapping.hpp:532
MemoryMapping m_mapping
Definition: memory_mapping.hpp:332
int flag_type
Definition: memory_mapping.hpp:132
MemoryMapping(size_t size, mapping_mode mode, int fd=-1, off_t offset=0)
Definition: memory_mapping.hpp:549
~TypedMemoryMapping() noexcept=default
size_t file_size(int fd)
Definition: file.hpp:67
flag_type get_flags() const noexcept
Definition: memory_mapping.hpp:539
int fd() const noexcept
Definition: memory_mapping.hpp:266
void unmap()
Definition: memory_mapping.hpp:581
Definition: memory_mapping.hpp:95
Definition: reader_iterator.hpp:39
mapping_mode
Definition: memory_mapping.hpp:98
void resize(size_t new_size)
Definition: memory_mapping.hpp:412
static size_t check_size(size_t size)
Definition: memory_mapping.hpp:139
int resize_fd(int fd)
Definition: memory_mapping.hpp:152
T * end()
Definition: memory_mapping.hpp:464
void * m_addr
The address where the memory is mapped.
Definition: memory_mapping.hpp:123
const T * cbegin() const
Definition: memory_mapping.hpp:468
off_t m_offset
Offset into the file.
Definition: memory_mapping.hpp:110
#define MAP_ANONYMOUS
Definition: memory_mapping.hpp:529
int m_fd
File handle we got the mapping from.
Definition: memory_mapping.hpp:113
size_t size() const noexcept
Definition: memory_mapping.hpp:257
mapping_mode m_mapping_mode
Mapping mode.
Definition: memory_mapping.hpp:116
Namespace for everything in the Osmium library.
Definition: assembler.hpp:73
AnonymousMemoryMapping(size_t size)
Definition: memory_mapping.hpp:306
Definition: memory_mapping.hpp:487
OSMIUM_DEPRECATED TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset=0)
Definition: memory_mapping.hpp:365
TypedMemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset=0)
Definition: memory_mapping.hpp:356
size_t size() const noexcept
Definition: memory_mapping.hpp:429
T * begin()
Definition: memory_mapping.hpp:455
Definition: memory_mapping.hpp:302
void make_invalid() noexcept
Definition: memory_mapping.hpp:521
AnonymousTypedMemoryMapping(size_t size)
Definition: memory_mapping.hpp:491
int fd() const noexcept
Definition: memory_mapping.hpp:439
void resize(size_t new_size)
Definition: memory_mapping.hpp:590
MemoryMapping & operator=(const MemoryMapping &)=delete
You can not copy a MemoryMapping.
Definition: memory_mapping.hpp:330
TypedMemoryMapping & operator=(const TypedMemoryMapping &)=delete
You can not copy a MemoryMapping.
void unmap()
Definition: memory_mapping.hpp:398
bool writable() const noexcept
Definition: memory_mapping.hpp:273
void resize_file(int fd, size_t new_size)
Definition: file.hpp:132
const T * begin() const
Definition: memory_mapping.hpp:476
size_t m_size
The size of the mapping.
Definition: memory_mapping.hpp:107
const T * cend() const
Definition: memory_mapping.hpp:472
TypedMemoryMapping(size_t size)
Definition: memory_mapping.hpp:342
OSMIUM_DEPRECATED MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0)
Definition: memory_mapping.hpp:190
T * get_addr() const
Definition: memory_mapping.hpp:283
bool writable() const noexcept
Definition: memory_mapping.hpp:446