Libosmium  2.4.1
Fast and flexible C++ library for working with OpenStreetMap data
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
reader.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_READER_HPP
2 #define OSMIUM_IO_READER_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <atomic>
37 #include <cerrno>
38 #include <cstdlib>
39 #include <fcntl.h>
40 #include <memory>
41 #include <string>
42 #include <system_error>
43 #include <thread>
44 #include <utility>
45 
46 #ifndef _WIN32
47 # include <sys/wait.h>
48 #endif
49 
50 #ifndef _MSC_VER
51 # include <unistd.h>
52 #endif
53 
55 #include <osmium/io/detail/input_format.hpp>
56 #include <osmium/io/detail/read_thread.hpp>
57 #include <osmium/io/detail/read_write.hpp>
58 #include <osmium/io/file.hpp>
59 #include <osmium/io/header.hpp>
60 #include <osmium/memory/buffer.hpp>
62 #include <osmium/thread/util.hpp>
63 #include <osmium/thread/queue.hpp>
64 
65 namespace osmium {
66 
67  namespace io {
68 
75  class Reader {
76 
78  osmium::io::detail::InputFormatFactory::create_input_type* m_input_format_creator;
80  std::atomic<bool> m_input_done;
82 
84 
85  std::unique_ptr<osmium::io::Decompressor> m_decompressor;
86  std::future<bool> m_read_future;
87 
88  std::unique_ptr<osmium::io::detail::InputFormat> m_input;
89 
90 #ifndef _WIN32
91 
102  static int execute(const std::string& command, const std::string& filename, int* childpid) {
103  int pipefd[2];
104  if (pipe(pipefd) < 0) {
105  throw std::system_error(errno, std::system_category(), "opening pipe failed");
106  }
107  pid_t pid = fork();
108  if (pid < 0) {
109  throw std::system_error(errno, std::system_category(), "fork failed");
110  }
111  if (pid == 0) { // child
112  // close all file descriptors except one end of the pipe
113  for (int i = 0; i < 32; ++i) {
114  if (i != pipefd[1]) {
115  ::close(i);
116  }
117  }
118  if (dup2(pipefd[1], 1) < 0) { // put end of pipe as stdout/stdin
119  exit(1);
120  }
121 
122  ::open("/dev/null", O_RDONLY); // stdin
123  ::open("/dev/null", O_WRONLY); // stderr
124  // hack: -g switches off globbing in curl which allows [] to be used in file names
125  // this is important for XAPI URLs
126  // in theory this execute() function could be used for other commands, but it is
127  // only used for curl at the moment, so this is okay.
128  if (::execlp(command.c_str(), command.c_str(), "-g", filename.c_str(), nullptr) < 0) {
129  exit(1);
130  }
131  }
132  // parent
133  *childpid = pid;
134  ::close(pipefd[1]);
135  return pipefd[0];
136  }
137 #endif
138 
147  static int open_input_file_or_url(const std::string& filename, int* childpid) {
148  std::string protocol = filename.substr(0, filename.find_first_of(':'));
149  if (protocol == "http" || protocol == "https" || protocol == "ftp" || protocol == "file") {
150 #ifndef _WIN32
151  return execute("curl", filename, childpid);
152 #else
153  throw std::runtime_error("Reading OSM files from the network currently not supported on Windows.");
154 #endif
155  } else {
156  return osmium::io::detail::open_for_reading(filename);
157  }
158  }
159 
160  public:
161 
172  m_file(file.check()),
173  m_input_format_creator(osmium::io::detail::InputFormatFactory::instance().get_creator_function(m_file)),
174  m_read_which_entities(read_which_entities),
175  m_input_done(false),
176  m_childpid(0),
177  m_input_queue(20, "raw_input"), // XXX
178  m_decompressor(m_file.buffer() ?
179  osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), m_file.buffer(), m_file.buffer_size()) :
180  osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), open_input_file_or_url(m_file.filename(), &m_childpid))),
181  m_read_future(std::async(std::launch::async, detail::ReadThread(m_input_queue, m_decompressor.get(), m_input_done))),
182  m_input((*m_input_format_creator)(m_file, m_read_which_entities, m_input_queue)) {
183  }
184 
185  explicit Reader(const std::string& filename, osmium::osm_entity_bits::type read_types = osmium::osm_entity_bits::all) :
186  Reader(osmium::io::File(filename), read_types) {
187  }
188 
189  explicit Reader(const char* filename, osmium::osm_entity_bits::type read_types = osmium::osm_entity_bits::all) :
190  Reader(osmium::io::File(filename), read_types) {
191  }
192 
193  Reader(const Reader&) = delete;
194  Reader& operator=(const Reader&) = delete;
195 
197  try {
198  close();
199  }
200  catch (...) {
201  }
202  }
203 
212  void close() {
213  // Signal to input child process that it should wrap up.
214  m_input_done = true;
215 
216  m_input->close();
217 
218 #ifndef _WIN32
219  if (m_childpid) {
220  int status;
221  pid_t pid = ::waitpid(m_childpid, &status, 0);
222 #pragma GCC diagnostic push
223 #pragma GCC diagnostic ignored "-Wold-style-cast"
224  if (pid < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) {
225  throw std::system_error(errno, std::system_category(), "subprocess returned error");
226  }
227 #pragma GCC diagnostic pop
228  m_childpid = 0;
229  }
230 #endif
231 
232  osmium::thread::wait_until_done(m_read_future);
233  }
234 
239  return m_input->header();
240  }
241 
253  // If an exception happened in the input thread, re-throw
254  // it in this (the main) thread.
256 
257  if (m_read_which_entities == osmium::osm_entity_bits::nothing || m_input_done) {
258  // If the caller didn't want anything but the header, it will
259  // always get an empty buffer here.
260  return osmium::memory::Buffer();
261  }
262 
263  // m_input->read() can return an invalid buffer to signal EOF,
264  // or a valid buffer with or without data. A valid buffer
265  // without data is not an error, it just means we have to
266  // keep getting the next buffer until there is one with data.
267  while (true) {
268  osmium::memory::Buffer buffer = m_input->read();
269  if (!buffer) {
270  m_input_done = true;
271  return buffer;
272  }
273  if (buffer.committed() > 0) {
274  return buffer;
275  }
276  }
277  }
278 
283  bool eof() const {
284  return m_input_done;
285  }
286 
287  }; // class Reader
288 
297  template <class... TArgs>
300 
301  Reader reader(std::forward<TArgs>(args)...);
302  while (osmium::memory::Buffer read_buffer = reader.read()) {
303  buffer.add_buffer(read_buffer);
304  buffer.commit();
305  }
306 
307  return buffer;
308  }
309 
310  } // namespace io
311 
312 } // namespace osmium
313 
314 #endif // OSMIUM_IO_READER_HPP
osmium::memory::Buffer read()
Definition: reader.hpp:252
Reader(const osmium::io::File &file, osmium::osm_entity_bits::type read_which_entities=osmium::osm_entity_bits::all)
Definition: reader.hpp:171
type
Definition: entity_bits.hpp:60
int m_childpid
Definition: reader.hpp:81
osmium::memory::Buffer read_file(TArgs &&...args)
Definition: reader.hpp:298
Definition: reader_iterator.hpp:39
std::unique_ptr< osmium::io::Decompressor > m_decompressor
Definition: reader.hpp:85
Reader & operator=(const Reader &)=delete
bool eof() const
Definition: reader.hpp:283
object or changeset
Definition: entity_bits.hpp:71
osmium::io::File m_file
Definition: reader.hpp:77
Definition: file.hpp:73
Namespace for everything in the Osmium library.
Definition: assembler.hpp:55
void add_buffer(const Buffer &buffer)
Definition: buffer.hpp:379
static int execute(const std::string &command, const std::string &filename, int *childpid)
Definition: reader.hpp:102
Definition: reader.hpp:75
osmium::io::Header header() const
Definition: reader.hpp:238
std::future< bool > m_read_future
Definition: reader.hpp:86
size_t committed() const noexcept
Definition: buffer.hpp:217
Definition: buffer.hpp:94
osmium::thread::Queue< std::string > m_input_queue
Definition: reader.hpp:83
static int open_input_file_or_url(const std::string &filename, int *childpid)
Definition: reader.hpp:147
Definition: entity_bits.hpp:62
~Reader()
Definition: reader.hpp:196
std::unique_ptr< osmium::io::detail::InputFormat > m_input
Definition: reader.hpp:88
Reader(const char *filename, osmium::osm_entity_bits::type read_types=osmium::osm_entity_bits::all)
Definition: reader.hpp:189
void check_for_exception(std::future< T > &future)
Definition: util.hpp:53
Definition: compression.hpp:105
osmium::io::detail::InputFormatFactory::create_input_type * m_input_format_creator
Definition: reader.hpp:78
Definition: header.hpp:49
std::atomic< bool > m_input_done
Definition: reader.hpp:80
void close()
Definition: reader.hpp:212
osmium::osm_entity_bits::type m_read_which_entities
Definition: reader.hpp:79
void wait_until_done(std::future< T > &future)
Definition: util.hpp:64
size_t commit()
Definition: buffer.hpp:272
Reader(const std::string &filename, osmium::osm_entity_bits::type read_types=osmium::osm_entity_bits::all)
Definition: reader.hpp:185