Libosmium  2.12.2
Fast and flexible C++ library for working with OpenStreetMap data
memory_mapping.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_UTIL_MEMORY_MAPPING_HPP
2 #define OSMIUM_UTIL_MEMORY_MAPPING_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2017 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <cassert>
37 #include <cerrno>
38 #include <cstddef>
39 #include <stdexcept>
40 #include <system_error>
41 
43 #include <osmium/util/file.hpp>
44 
45 #ifndef _WIN32
46 # include <sys/mman.h>
47 #else
48 # include <fcntl.h>
49 # include <io.h>
50 # include <windows.h>
51 # include <sys/types.h>
52 #endif
53 
54 namespace osmium {
55 
56  namespace util {
57 
95  class MemoryMapping {
96 
97  public:
98 
99  enum class mapping_mode {
100  readonly = 0,
101  write_private = 1,
102  write_shared = 2
103  };
104 
105  private:
106 
108  size_t m_size;
109 
111  off_t m_offset;
112 
114  int m_fd;
115 
118 
119 #ifdef _WIN32
120  HANDLE m_handle;
121 #endif
122 
124  void* m_addr;
125 
126  bool is_valid() const noexcept;
127 
128  void make_invalid() noexcept;
129 
130 #ifdef _WIN32
131  using flag_type = DWORD;
132 #else
133  using flag_type = int;
134 #endif
135 
136  flag_type get_protection() const noexcept;
137 
138  flag_type get_flags() const noexcept;
139 
140  static size_t check_size(size_t size) {
141  if (size == 0) {
142  throw std::runtime_error("Zero-sized mapping is not allowed.");
143  }
144  return size;
145  }
146 
147 #ifdef _WIN32
148  HANDLE get_handle() const noexcept;
149  HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept;
150  void* osmium::util::MemoryMapping::map_view_of_file() const noexcept;
151 #endif
152 
153  int resize_fd(int fd) {
154  // Anonymous mapping doesn't need resizing.
155  if (fd == -1) {
156  return -1;
157  }
158 
159  // Make sure the file backing this mapping is large enough.
160  if (osmium::util::file_size(fd) < m_size + m_offset) {
161  osmium::util::resize_file(fd, m_size + m_offset);
162  }
163  return fd;
164  }
165 
166  public:
167 
184  MemoryMapping(size_t size, mapping_mode mode, int fd=-1, off_t offset=0);
185 
191  OSMIUM_DEPRECATED MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0) :
192  MemoryMapping(size, writable ? mapping_mode::write_shared : mapping_mode::readonly, fd, offset) {
193  }
194 
196  MemoryMapping(const MemoryMapping&) = delete;
197 
199  MemoryMapping& operator=(const MemoryMapping&) = delete;
200 
205  MemoryMapping(MemoryMapping&& other);
206 
211 
216  ~MemoryMapping() noexcept {
217  try {
218  unmap();
219  } catch (const std::system_error&) {
220  // Ignore any exceptions because destructor must not throw.
221  }
222  }
223 
230  void unmap();
231 
243  void resize(size_t new_size);
244 
249  explicit operator bool() const noexcept {
250  return is_valid();
251  }
252 
258  size_t size() const noexcept {
259  return m_size;
260  }
261 
267  int fd() const noexcept {
268  return m_fd;
269  }
270 
274  bool writable() const noexcept {
275  return m_mapping_mode != mapping_mode::readonly;
276  }
277 
283  template <typename T = void>
284  T* get_addr() const {
285  if (is_valid()) {
286  return reinterpret_cast<T*>(m_addr);
287  }
288  throw std::runtime_error("invalid memory mapping");
289  }
290 
291  }; // class MemoryMapping
292 
304 
305  public:
306 
307  explicit AnonymousMemoryMapping(size_t size) :
309  }
310 
311 #ifndef __linux__
312 
316  void resize(size_t) = delete;
317 #endif
318 
319  }; // class AnonymousMemoryMapping
320 
330  template <typename T>
332 
334 
335  public:
336 
343  explicit TypedMemoryMapping(size_t size) :
344  m_mapping(sizeof(T) * size, MemoryMapping::mapping_mode::write_private) {
345  }
346 
357  TypedMemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset = 0) :
358  m_mapping(sizeof(T) * size, mode, fd, sizeof(T) * offset) {
359  }
360 
366  OSMIUM_DEPRECATED TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset = 0) :
367  m_mapping(sizeof(T) * size, writable ? MemoryMapping::mapping_mode::write_shared : MemoryMapping::mapping_mode::readonly, fd, sizeof(T) * offset) {
368  }
369 
371  TypedMemoryMapping(const TypedMemoryMapping&) = delete;
372 
375 
380  TypedMemoryMapping(TypedMemoryMapping&& other) = default;
381 
385  TypedMemoryMapping& operator=(TypedMemoryMapping&& other) = default;
386 
391  ~TypedMemoryMapping() noexcept = default;
392 
399  void unmap() {
400  m_mapping.unmap();
401  }
402 
413  void resize(size_t new_size) {
414  m_mapping.resize(sizeof(T) * new_size);
415  }
416 
421  explicit operator bool() const noexcept {
422  return !!m_mapping;
423  }
424 
430  size_t size() const noexcept {
431  assert(m_mapping.size() % sizeof(T) == 0);
432  return m_mapping.size() / sizeof(T);
433  }
434 
440  int fd() const noexcept {
441  return m_mapping.fd();
442  }
443 
447  bool writable() const noexcept {
448  return m_mapping.writable();
449  }
450 
456  T* begin() {
457  return m_mapping.get_addr<T>();
458  }
459 
465  T* end() {
466  return m_mapping.get_addr<T>() + size();
467  }
468 
469  const T* cbegin() const {
470  return m_mapping.get_addr<T>();
471  }
472 
473  const T* cend() const {
474  return m_mapping.get_addr<T>() + size();
475  }
476 
477  const T* begin() const {
478  return m_mapping.get_addr<T>();
479  }
480 
481  const T* end() const {
482  return m_mapping.get_addr<T>() + size();
483  }
484 
485  }; // class TypedMemoryMapping
486 
487  template <typename T>
489 
490  public:
491 
493  TypedMemoryMapping<T>(size) {
494  }
495 
496 #ifndef __linux__
497 
501  void resize(size_t) = delete;
502 #endif
503 
504  }; // class AnonymousTypedMemoryMapping
505 
506  } // namespace util
507 
508 } // namespace osmium
509 
510 #ifndef _WIN32
511 
512 // =========== Unix implementation =============
513 
514 // MAP_FAILED is often a macro containing an old style cast
515 #pragma GCC diagnostic push
516 #pragma GCC diagnostic ignored "-Wold-style-cast"
517 
518 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
519  return m_addr != MAP_FAILED;
520 }
521 
523  m_addr = MAP_FAILED;
524 }
525 
526 #pragma GCC diagnostic pop
527 
528 // for BSD systems
529 #ifndef MAP_ANONYMOUS
530 # define MAP_ANONYMOUS MAP_ANON
531 #endif
532 
533 inline int osmium::util::MemoryMapping::get_protection() const noexcept {
535  return PROT_READ;
536  }
537  return PROT_READ | PROT_WRITE;
538 }
539 
540 inline int osmium::util::MemoryMapping::get_flags() const noexcept {
541  if (m_fd == -1) {
542  return MAP_PRIVATE | MAP_ANONYMOUS;
543  }
545  return MAP_SHARED;
546  }
547  return MAP_PRIVATE;
548 }
549 
550 inline osmium::util::MemoryMapping::MemoryMapping(size_t size, mapping_mode mode, int fd, off_t offset) :
551  m_size(check_size(size)),
552  m_offset(offset),
553  m_fd(resize_fd(fd)),
554  m_mapping_mode(mode),
555  m_addr(::mmap(nullptr, m_size, get_protection(), get_flags(), m_fd, m_offset)) {
556  assert(!(fd == -1 && mode == mapping_mode::readonly));
557  if (!is_valid()) {
558  throw std::system_error(errno, std::system_category(), "mmap failed");
559  }
560 }
561 
563  m_size(other.m_size),
564  m_offset(other.m_offset),
565  m_fd(other.m_fd),
567  m_addr(other.m_addr) {
568  other.make_invalid();
569 }
570 
572  unmap();
573  m_size = other.m_size;
574  m_offset = other.m_offset;
575  m_fd = other.m_fd;
576  m_mapping_mode = other.m_mapping_mode;
577  m_addr = other.m_addr;
578  other.make_invalid();
579  return *this;
580 }
581 
583  if (is_valid()) {
584  if (::munmap(m_addr, m_size) != 0) {
585  throw std::system_error(errno, std::system_category(), "munmap failed");
586  }
587  make_invalid();
588  }
589 }
590 
591 inline void osmium::util::MemoryMapping::resize(size_t new_size) {
592  assert(new_size > 0 && "can not resize to zero size");
593  if (m_fd == -1) { // anonymous mapping
594 #ifdef __linux__
595  m_addr = ::mremap(m_addr, m_size, new_size, MREMAP_MAYMOVE);
596  if (!is_valid()) {
597  throw std::system_error(errno, std::system_category(), "mremap failed");
598  }
599  m_size = new_size;
600 #else
601  assert(false && "can't resize anonymous mappings on non-linux systems");
602 #endif
603  } else { // file-based mapping
604  unmap();
605  m_size = new_size;
606  resize_fd(m_fd);
607  m_addr = ::mmap(nullptr, new_size, get_protection(), get_flags(), m_fd, m_offset);
608  if (!is_valid()) {
609  throw std::system_error(errno, std::system_category(), "mmap (remap) failed");
610  }
611  }
612 }
613 
614 #else
615 
616 // =========== Windows implementation =============
617 
618 /* References:
619  * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx
620  * CloseHandle: http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx
621  * MapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx
622  * UnmapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx
623  */
624 
625 namespace osmium {
626 
627  namespace util {
628 
629  inline DWORD dword_hi(uint64_t x) {
630  return static_cast<DWORD>(x >> 32);
631  }
632 
633  inline DWORD dword_lo(uint64_t x) {
634  return static_cast<DWORD>(x & 0xffffffff);
635  }
636 
637  } // namespace util
638 
639 } // namespace osmium
640 
641 inline DWORD osmium::util::MemoryMapping::get_protection() const noexcept {
642  switch (m_mapping_mode) {
644  return PAGE_READONLY;
646  return PAGE_WRITECOPY;
647  default: // mapping_mode::write_shared
648  break;
649  }
650  return PAGE_READWRITE;
651 }
652 
653 inline DWORD osmium::util::MemoryMapping::get_flags() const noexcept {
654  switch (m_mapping_mode) {
656  return FILE_MAP_READ;
658  return FILE_MAP_COPY;
659  default: // mapping_mode::write_shared
660  break;
661  }
662  return FILE_MAP_WRITE;
663 }
664 
665 inline HANDLE osmium::util::MemoryMapping::get_handle() const noexcept {
666  if (m_fd == -1) {
667  return INVALID_HANDLE_VALUE;
668  }
669  return reinterpret_cast<HANDLE>(_get_osfhandle(m_fd));
670 }
671 
672 inline HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept {
673  if (m_fd != -1) {
674  _setmode(m_fd, _O_BINARY);
675  }
676  return CreateFileMapping(get_handle(), nullptr, get_protection(), osmium::util::dword_hi(static_cast<uint64_t>(m_size) + m_offset), osmium::util::dword_lo(static_cast<uint64_t>(m_size) + m_offset), nullptr);
677 }
678 
679 inline void* osmium::util::MemoryMapping::map_view_of_file() const noexcept {
680  return MapViewOfFile(m_handle, get_flags(), osmium::util::dword_hi(m_offset), osmium::util::dword_lo(m_offset), m_size);
681 }
682 
683 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
684  return m_addr != nullptr;
685 }
686 
687 inline void osmium::util::MemoryMapping::make_invalid() noexcept {
688  m_addr = nullptr;
689 }
690 
691 inline osmium::util::MemoryMapping::MemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset) :
692  m_size(check_size(size)),
693  m_offset(offset),
694  m_fd(resize_fd(fd)),
695  m_mapping_mode(mode),
696  m_handle(create_file_mapping()),
697  m_addr(nullptr) {
698 
699  if (!m_handle) {
700  throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed");
701  }
702 
703  m_addr = map_view_of_file();
704  if (!is_valid()) {
705  throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed");
706  }
707 }
708 
710  m_size(other.m_size),
711  m_offset(other.m_offset),
712  m_fd(other.m_fd),
713  m_mapping_mode(other.m_mapping_mode),
714  m_handle(std::move(other.m_handle)),
715  m_addr(other.m_addr) {
716  other.make_invalid();
717  other.m_handle = nullptr;
718 }
719 
721  unmap();
722  m_size = other.m_size;
723  m_offset = other.m_offset;
724  m_fd = other.m_fd;
725  m_mapping_mode = other.m_mapping_mode;
726  m_handle = std::move(other.m_handle);
727  m_addr = other.m_addr;
728  other.make_invalid();
729  other.m_handle = nullptr;
730  return *this;
731 }
732 
734  if (is_valid()) {
735  if (! UnmapViewOfFile(m_addr)) {
736  throw std::system_error(GetLastError(), std::system_category(), "UnmapViewOfFile failed");
737  }
738  make_invalid();
739  }
740 
741  if (m_handle) {
742  if (! CloseHandle(m_handle)) {
743  throw std::system_error(GetLastError(), std::system_category(), "CloseHandle failed");
744  }
745  m_handle = nullptr;
746  }
747 }
748 
749 inline void osmium::util::MemoryMapping::resize(size_t new_size) {
750  unmap();
751 
752  m_size = new_size;
753  resize_fd(m_fd);
754 
755  m_handle = create_file_mapping();
756  if (!m_handle) {
757  throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed");
758  }
759 
760  m_addr = map_view_of_file();
761  if (!is_valid()) {
762  throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed");
763  }
764 }
765 
766 #endif
767 
768 #endif // OSMIUM_UTIL_MEMORY_MAPPING_HPP
~MemoryMapping() noexcept
Definition: memory_mapping.hpp:216
bool is_valid() const noexcept
Definition: memory_mapping.hpp:518
#define OSMIUM_DEPRECATED
Definition: compatibility.hpp:50
flag_type get_protection() const noexcept
Definition: memory_mapping.hpp:533
MemoryMapping m_mapping
Definition: memory_mapping.hpp:333
int flag_type
Definition: memory_mapping.hpp:133
MemoryMapping(size_t size, mapping_mode mode, int fd=-1, off_t offset=0)
Definition: memory_mapping.hpp:550
size_t file_size(int fd)
Definition: file.hpp:70
const T * begin() const
Definition: memory_mapping.hpp:477
flag_type get_flags() const noexcept
Definition: memory_mapping.hpp:540
int fd() const noexcept
Definition: memory_mapping.hpp:267
void unmap()
Definition: memory_mapping.hpp:582
Definition: memory_mapping.hpp:95
mapping_mode
Definition: memory_mapping.hpp:99
void resize(size_t new_size)
Definition: memory_mapping.hpp:413
static size_t check_size(size_t size)
Definition: memory_mapping.hpp:140
int resize_fd(int fd)
Definition: memory_mapping.hpp:153
T * end()
Definition: memory_mapping.hpp:465
void * m_addr
The address where the memory is mapped.
Definition: memory_mapping.hpp:124
off_t m_offset
Offset into the file.
Definition: memory_mapping.hpp:111
#define MAP_ANONYMOUS
Definition: memory_mapping.hpp:530
int m_fd
File handle we got the mapping from.
Definition: memory_mapping.hpp:114
size_t size() const noexcept
Definition: memory_mapping.hpp:258
mapping_mode m_mapping_mode
Mapping mode.
Definition: memory_mapping.hpp:117
Namespace for everything in the Osmium library.
Definition: assembler.hpp:63
AnonymousMemoryMapping(size_t size)
Definition: memory_mapping.hpp:307
Definition: memory_mapping.hpp:488
OSMIUM_DEPRECATED TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset=0)
Definition: memory_mapping.hpp:366
TypedMemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset=0)
Definition: memory_mapping.hpp:357
size_t size() const noexcept
Definition: memory_mapping.hpp:430
T * begin()
Definition: memory_mapping.hpp:456
Definition: memory_mapping.hpp:303
void make_invalid() noexcept
Definition: memory_mapping.hpp:522
const T * cbegin() const
Definition: memory_mapping.hpp:469
AnonymousTypedMemoryMapping(size_t size)
Definition: memory_mapping.hpp:492
int fd() const noexcept
Definition: memory_mapping.hpp:440
void resize(size_t new_size)
Definition: memory_mapping.hpp:591
MemoryMapping & operator=(const MemoryMapping &)=delete
You can not copy a MemoryMapping.
Definition: memory_mapping.hpp:331
const T * end() const
Definition: memory_mapping.hpp:481
void unmap()
Definition: memory_mapping.hpp:399
bool writable() const noexcept
Definition: memory_mapping.hpp:274
void resize_file(int fd, size_t new_size)
Definition: file.hpp:135
size_t m_size
The size of the mapping.
Definition: memory_mapping.hpp:108
T * get_addr() const
Definition: memory_mapping.hpp:284
TypedMemoryMapping(size_t size)
Definition: memory_mapping.hpp:343
OSMIUM_DEPRECATED MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0)
Definition: memory_mapping.hpp:191
const T * cend() const
Definition: memory_mapping.hpp:473
bool writable() const noexcept
Definition: memory_mapping.hpp:447