MRPT  1.9.9
net_utils.cpp
Go to the documentation of this file.
1 /* +------------------------------------------------------------------------+
2  | Mobile Robot Programming Toolkit (MRPT) |
3  | https://www.mrpt.org/ |
4  | |
5  | Copyright (c) 2005-2019, Individual contributors, see AUTHORS file |
6  | See: https://www.mrpt.org/Authors - All rights reserved. |
7  | Released under BSD License. See: https://www.mrpt.org/License |
8  +------------------------------------------------------------------------+ */
9 
10 #include "comms-precomp.h" // Precompiled headers
11 
14 #include <mrpt/comms/net_utils.h>
15 #include <mrpt/core/exceptions.h>
16 #include <mrpt/core/format.h>
17 #include <mrpt/system/CTicTac.h>
19 
20 #include <cstdio>
21 #include <cstring>
22 #include <future>
23 #include <thread>
24 
25 #if defined(MRPT_OS_LINUX) || defined(__APPLE__)
26 #define INVALID_SOCKET (-1)
27 #include <arpa/inet.h>
28 #include <fcntl.h>
29 #include <netdb.h>
30 #include <netinet/in.h>
31 #include <sys/ioctl.h>
32 #include <sys/socket.h>
33 #include <sys/types.h>
34 #include <unistd.h>
35 #include <cerrno>
36 #endif
37 
38 #ifdef _WIN32
39 #include <winsock.h>
40 #endif
41 
42 using namespace mrpt;
43 using namespace mrpt::system;
44 using namespace mrpt::comms;
45 using namespace mrpt::comms::net;
46 using namespace std;
47 
48 /*---------------------------------------------------------------
49  http_get
50  ---------------------------------------------------------------*/
52  const string& url, string& out_content, string& out_errormsg, int port,
53  const string& auth_user, const string& auth_pass,
54  int* out_http_responsecode,
55  mrpt::system::TParameters<string>* extra_headers,
56  mrpt::system::TParameters<string>* out_headers, int timeout_ms)
57 {
58  std::vector<uint8_t> data;
60  url, data, out_errormsg, port, auth_user, auth_pass,
61  out_http_responsecode, extra_headers, out_headers, timeout_ms);
62 
63  out_content.resize(data.size());
64  if (!data.empty()) std::memcpy(&out_content[0], &data[0], data.size());
65 
66  return ret;
67 }
68 
70  const string& http_method, const string& http_send_content,
71  const string& url, std::vector<uint8_t>& out_content, string& out_errormsg,
72  int port, const string& auth_user, const string& auth_pass,
73  int* out_http_responsecode,
74  mrpt::system::TParameters<string>* extra_headers,
75  mrpt::system::TParameters<string>* out_headers, int timeout_ms)
76 {
77  // Reset output data:
78  out_content.clear();
79  if (out_http_responsecode) *out_http_responsecode = 0;
80  if (out_headers) out_headers->clear();
81 
82  // URL must be:
83  // http://<SERVER>/<LOCAL_ADDR>
84 
85  if (0 != ::strncmp(url.c_str(), "http://", 7))
86  {
87  out_errormsg = "URL must start with 'http://'";
88  return net::erBadURL;
89  }
90 
91  string server_addr = url.substr(7);
92  string get_object = "/";
93 
94  // Remove from the first "/" on:
95  size_t pos = server_addr.find("/");
96  if (pos == 0)
97  {
98  out_errormsg = "Server name not found in URL";
99  return net::erBadURL;
100  }
101  if (pos != string::npos)
102  {
103  get_object = server_addr.substr(pos);
104  server_addr.resize(pos);
105  }
106 
107  CClientTCPSocket sock;
108 
109  try
110  {
111  // Connect:
112  sock.connect(server_addr, port, timeout_ms);
113  }
114  catch (const std::exception& e)
115  {
116  out_errormsg = e.what();
117  return net::erCouldntConnect;
118  }
119 
120  try
121  {
122  // Set the user-defined headers (we may overwrite them if needed)
123  TParameters<string> headers_to_send;
124  if (extra_headers) headers_to_send = *extra_headers;
125 
126  headers_to_send["Connection"] = "close"; // Don't keep alive
127 
128  if (!headers_to_send.has("User-Agent"))
129  headers_to_send["User-Agent"] = "MRPT Library";
130 
131  // Implement HTTP Basic authentication:
132  // See: http://en.wikipedia.org/wiki/Basic_access_authentication
133  if (!auth_user.empty())
134  {
135  string auth_str = auth_user + string(":") + auth_pass;
136  std::vector<uint8_t> v(auth_str.size());
137  std::memcpy(&v[0], &auth_str[0], auth_str.size());
138 
139  string encoded_str;
140  mrpt::system::encodeBase64(v, encoded_str);
141 
142  headers_to_send["Authorization"] = string("Basic ") + encoded_str;
143  }
144 
145  if (!http_send_content.empty() &&
146  headers_to_send.find("Content-Length") == headers_to_send.end())
147  {
148  headers_to_send["Content-Length"] =
149  mrpt::format("%u", (unsigned int)http_send_content.size());
150  }
151 
152  // Prepare the request string
153  // ---------------------------------
154  string req = format(
155  "%s %s HTTP/1.1\r\n"
156  "Host: %s\r\n",
157  http_method.c_str(), get_object.c_str(), server_addr.c_str());
158 
159  // Other headers:
160  for (auto i = headers_to_send.begin(); i != headers_to_send.end(); ++i)
161  {
162  req += i->first;
163  req += ": ";
164  req += i->second;
165  req += "\r\n";
166  }
167 
168  // End:
169  req += "\r\n";
170 
171  // Any POST data?
172  req += http_send_content;
173 
174  // Send:
175  sock.sendString(req);
176 
177  // Read answer:
178  std::vector<uint8_t> buf;
179  buf.reserve(1 << 14);
180 
181  size_t total_read = 0;
182  bool content_length_read = false;
183  bool all_headers_read = false;
184  size_t content_length = 0;
185  size_t content_offset = 0;
186  int http_code = 0;
188 
189  CTicTac watchdog;
190  watchdog.Tic();
191 
192  while (!content_length_read ||
193  total_read < (content_offset + content_length))
194  {
195  // Read until "Content-Length: XXX \r\n" is read or the whole
196  // message is read,
197  // or an error code is read.
198  size_t to_read_now;
199 
200  if (!content_length_read)
201  {
202  to_read_now = 1500;
203  }
204  else
205  {
206  to_read_now = (content_length + content_offset) - total_read;
207  }
208 
209  // make room for the data to come:
210  buf.resize(total_read + to_read_now + 1);
211 
212  // Read:
213  size_t len =
214  sock.readAsync(&buf[total_read], to_read_now, timeout_ms, 100);
215  if (!len)
216  {
217  //
218  if (!sock.isConnected())
219  {
220  if (all_headers_read) // It seems we're done...
221  break;
222  else
223  {
224  out_errormsg = "Connection to server was lost";
225  return net::erCouldntConnect;
226  }
227  }
228 
229  if (watchdog.Tac() > 1e-3 * timeout_ms)
230  {
231  out_errormsg = "Timeout waiting answer from server";
232  return net::erCouldntConnect;
233  }
234  std::this_thread::sleep_for(10ms);
235  continue;
236  }
237  total_read += len;
238  watchdog.Tic();
239 
240  buf[total_read] = '\0';
241 
242  // do we have a \r\n\r\n ??
243  if (!all_headers_read)
244  {
245  const char* ptr = ::strstr(
246  reinterpret_cast<const char*>(&buf[0]), "\r\n\r\n");
247  if (ptr)
248  {
249  all_headers_read = true;
250  const size_t pos_dblret = ((char*)ptr) - (char*)(&buf[0]);
251 
252  // Process the headers:
253  // ------------------------------
254  if (!::strncmp("HTTP/", (const char*)&buf[0], 5))
255  {
256  http_code = ::atoi((const char*)&buf[9]);
257  }
258  else
259  {
260  // May it be a "SOURCETABLE " answer for NTRIP
261  // protocol??
262  if (!::strncmp(
263  "SOURCETABLE ", (const char*)&buf[0], 12))
264  {
265  http_code = ::atoi((const char*)&buf[12]);
266  }
267  else
268  {
269  out_errormsg =
270  "Server didn't send an HTTP/1.1 answer.";
271  return net::erOtherHTTPError;
272  }
273  }
274 
275  // Check the HTTP code and the content-length:
276  content_offset = pos_dblret + 4;
277 
278  // Do we have a "Content-Length:"??
279  const char* ptr_len = ::strstr(
280  reinterpret_cast<const char*>(&buf[0]),
281  "Content-Length:");
282  if (ptr_len)
283  {
284  content_length = ::atol(ptr_len + 15);
285  content_length_read = true;
286  }
287 
288  // Parse the rest of HTTP headers:
289  {
290  string aux_all_headers;
291  deque<string> lstLines;
292  aux_all_headers.resize(content_offset);
293  std::memcpy(
294  &aux_all_headers[0], &buf[0], content_offset);
295 
297  aux_all_headers, "\r\n", lstLines);
298 
299  for (auto i = lstLines.begin(); i != lstLines.end();
300  ++i)
301  {
302  const size_t p = i->find(":");
303  if (p == string::npos) continue;
304 
305  const string key = i->substr(0, p);
306  const string val = i->substr(p + 2);
307  rx_headers[key] = val;
308  }
309  }
310  }
311  }
312  } // end while
313 
314  if (out_http_responsecode) *out_http_responsecode = http_code;
315  if (out_headers) *out_headers = rx_headers;
316 
317  // Remove the headers from the content:
318  buf.erase(buf.begin(), buf.begin() + content_offset);
319 
320  // Process: "Transfer-Encoding: chunked"
321  if (rx_headers.has("Transfer-Encoding") &&
322  rx_headers["Transfer-Encoding"] == "chunked")
323  {
324  // See: http://en.wikipedia.org/wiki/Chunked_transfer_encoding
325 
326  size_t index = 0;
327  while (index < buf.size())
328  {
329  if (buf[index] == '\r' && buf[index + 1] == '\n')
330  {
331  buf.erase(buf.begin() + index, buf.begin() + index + 2);
332  continue;
333  }
334 
335  const char* pCRLF = ::strstr((const char*)&buf[index], "\r\n");
336  if (!pCRLF) break;
337 
338  const size_t len_substr = ((char*)pCRLF) - (char*)(&buf[index]);
339 
340  string sLen((const char*)&buf[index], len_substr);
341  sLen = string("0x") + sLen;
342 
343  unsigned int lenChunk;
344  int fields = ::sscanf(sLen.c_str(), "%x", &lenChunk);
345  if (!fields) break;
346 
347  // Remove the len of this chunk header from the data:
348  buf.erase(
349  buf.begin() + index, buf.begin() + index + len_substr + 2);
350 
351  index += lenChunk;
352 
353  if (!lenChunk)
354  {
355  buf.resize(index);
356  break;
357  }
358  }
359  }
360 
361  // Set the content output:
362  out_content.swap(buf);
363 
364  if (http_code == 200)
365  {
366  return net::erOk;
367  }
368  else
369  {
370  out_errormsg = format("HTTP error %i", http_code);
371  return net::erOtherHTTPError;
372  }
373  }
374  catch (const std::exception& e)
375  {
376  out_errormsg = e.what();
377  return net::erCouldntConnect;
378  }
379 
380  return net::erOk;
381 }
382 
383 /*---------------------------------------------------------------
384  http_get
385 ---------------------------------------------------------------*/
387  const string& url, std::vector<uint8_t>& out_content, string& out_errormsg,
388  int port, const string& auth_user, const string& auth_pass,
389  int* out_http_responsecode,
390  mrpt::system::TParameters<string>* extra_headers,
391  mrpt::system::TParameters<string>* out_headers, int timeout_ms)
392 {
393  return http_request(
394  "GET", "", url, out_content, out_errormsg, port, auth_user, auth_pass,
395  out_http_responsecode, extra_headers, out_headers, timeout_ms);
396 }
397 
398 /** Resolve a server address by its name, returning its IP address as a string -
399  * This method has a timeout for the maximum time to wait for the DNS server.
400  * For example: server_name="www.google.com" -> out_ip="209.85.227.99"
401  *
402  * \return true on success, false on timeout or other error.
403  */
405  const std::string& server_name, std::string& out_ip,
406  const unsigned int timeout_ms)
407 {
408  // Firstly: If it's a numeric address already, do nothing:
409  if (server_name.find_first_not_of("0123456789. ") == std::string::npos)
410  {
411  // It's a pure IP address:
412  out_ip = server_name;
413  return true;
414  }
415 
416  // Solve DNS --------------
417  // It seems that the only reliable way of *with a timeout* is to launch a
418  // separate thread.
419 
420  std::future<std::string> dns_result_fut =
421  std::async(std::launch::async, [&]() {
422 // Windows-specific stuff:
423 #ifdef _WIN32
424  {
425  // Init the WinSock Library:
426  WORD wVersionRequested = MAKEWORD(2, 0);
427  WSADATA wsaData;
428  if (WSAStartup(wVersionRequested, &wsaData))
429  {
430  std::cerr
431  << "thread_DNS_solver_async: Error calling WSAStartup";
432  return std::string();
433  }
434  }
435 #endif
436 
437  // Do the DNS lookup:
438  std::string dns_result;
439 
440  hostent* he = gethostbyname(server_name.c_str());
441  if (!he)
442  {
443  dns_result.clear(); // empty string -> error.
444  }
445  else
446  {
447  struct in_addr ADDR;
448  std::memcpy(
449  &ADDR, he->h_addr,
450  sizeof(ADDR)); // Was: *((struct in_addr *)he->h_addr);
451  // Convert address to text:
452  dns_result = string(inet_ntoa(ADDR));
453  }
454 
455 #ifdef _WIN32
456  WSACleanup();
457 #endif
458  return dns_result;
459  });
460 
461  auto status =
462  dns_result_fut.wait_for(std::chrono::milliseconds(timeout_ms));
463 
464  if (status == std::future_status::ready)
465  {
466  // Done: Anyway, it can still be an error result:
467  out_ip = dns_result_fut.get();
468  return !out_ip.empty();
469  }
470  else
471  {
472  // Timeout:
473  out_ip.clear();
474 
475  return false;
476  }
477 }
478 
479 /** Returns a description of the last Sockets error */
481 {
482 #ifdef _WIN32
483  const int errnum = WSAGetLastError();
484  char* s = nullptr;
485  FormatMessageA(
486  FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
487  FORMAT_MESSAGE_IGNORE_INSERTS,
488  nullptr, errnum, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&s,
489  0, nullptr);
490  const std::string str = mrpt::format("%s [errno=%d]", s, errnum);
491  LocalFree(s);
492  return str;
493 #else
494  return std::string(strerror(errno));
495 #endif
496 }
497 
499  const std::string& address, const int max_attempts,
500  std::string* output_str /*=NULL*/)
501 {
502  using namespace std;
503 
504  // Format a command string
505  string cmd_str = "ping";
506 
507 // different "count" argument for Windows and *NIX systems
508 #if defined(MRPT_OS_LINUX) || defined(__APPLE__)
509  cmd_str += " -c ";
510 #else
511  cmd_str += " -n ";
512 #endif
513  cmd_str += std::to_string(max_attempts);
514 
515  // Address:
516  cmd_str += " ";
517  cmd_str += address;
518 
519 // Redirection:
520 #if defined(MRPT_OS_LINUX) || defined(__APPLE__)
521  cmd_str += " 2>&1";
522 #endif
523 
524  // Finally exec the command
525  int code = executeCommand(cmd_str, output_str);
526 
527  return (code == 0);
528 }
double Tac() noexcept
Stops the stopwatch.
Definition: CTicTac.cpp:86
iterator begin() noexcept
Definition: TParameters.h:134
std::string to_string(T v)
Just like std::to_string(), but with an overloaded version for std::string arguments.
Definition: format.h:36
void connect(const std::string &remotePartAddress, unsigned short remotePartTCPPort, unsigned int timeout_ms=0)
Establishes a connection with a remote part.
std::string std::string format(std::string_view fmt, ARGS &&... args)
Definition: format.h:26
bool isConnected()
Returns true if this objects represents a successfully connected socket.
A high-performance stopwatch, with typical resolution of nanoseconds.
std::string getLastSocketErrorStr()
Returns a description of the last Sockets error.
Definition: net_utils.cpp:480
iterator end() noexcept
Definition: TParameters.h:136
STL namespace.
GLdouble s
Definition: glext.h:3682
GLenum GLsizei len
Definition: glext.h:4756
ERRORCODE_HTTP http_get(const string &url, std::vector< uint8_t > &out_content, string &out_errormsg, int port=80, const string &auth_user=string(), const string &auth_pass=string(), int *out_http_responsecode=nullptr, mrpt::system::TParameters< string > *extra_headers=nullptr, mrpt::system::TParameters< string > *out_headers=nullptr, int timeout_ms=1000)
Perform an HTTP GET operation (version for retrieving the data as a std::vector<uint8_t>) ...
Definition: net_utils.cpp:386
void tokenize(const std::string &inString, const std::string &inDelimiters, OUT_CONTAINER &outTokens, bool skipBlankTokens=true) noexcept
Tokenizes a string according to a set of delimiting characters.
bool Ping(const std::string &address, const int max_attempts, std::string *output_str=nullptr)
Ping an IP address.
Definition: net_utils.cpp:498
ERRORCODE_HTTP
Possible returns from a HTTP request.
Definition: net_utils.h:31
bool has(const std::string &s) const
Definition: TParameters.h:69
bool DNS_resolve_async(const std::string &server_name, std::string &out_ip, const unsigned int timeout_ms=3000)
Resolve a server address by its name, returning its IP address as a string - This method has a timeou...
Definition: net_utils.cpp:404
GLuint index
Definition: glext.h:4068
int val
Definition: mrpt_jpeglib.h:957
GLsizei const GLchar ** string
Definition: glext.h:4116
iterator find(const std::string &key)
Definition: TParameters.h:132
int executeCommand(const std::string &command, std::string *output=nullptr, const std::string &mode="r")
Execute Generic Shell Command.
Definition: os.cpp:662
const GLdouble * v
Definition: glext.h:3684
This is the global namespace for all Mobile Robot Programming Toolkit (MRPT) libraries.
Definition: inftrees.h:28
void encodeBase64(const std::vector< uint8_t > &inputData, std::string &outString)
Encode a sequence of bytes as a string in base-64.
Definition: base64.cpp:29
A TCP socket that can be connected to a TCP server, implementing MRPT&#39;s CStream interface for passing...
ERRORCODE_HTTP http_request(const string &http_method, const string &http_send_content, const string &url, std::vector< uint8_t > &out_content, string &out_errormsg, int port=80, const string &auth_user=string(), const string &auth_pass=string(), int *out_http_responsecode=nullptr, mrpt::system::TParameters< string > *extra_headers=nullptr, mrpt::system::TParameters< string > *out_headers=nullptr, int timeout_ms=1000)
Generic function for HTTP GET & POST methods.
Definition: net_utils.cpp:69
A set of useful routines for networking.
Definition: net_utils.h:23
GLuint address
Definition: glext.h:7060
Serial and networking devices and utilities.
void Tic() noexcept
Starts the stopwatch.
Definition: CTicTac.cpp:75
size_t readAsync(void *Buffer, const size_t Count, const int timeoutStart_ms=-1, const int timeoutBetween_ms=-1)
A method for reading from the socket with an optional timeout.
For usage when passing a dynamic number of (numeric) arguments to a function, by name.
Definition: TParameters.h:54
GLsizei GLsizei GLenum GLenum const GLvoid * data
Definition: glext.h:3550
GLfloat GLfloat p
Definition: glext.h:6398
void sendString(const std::string &str)
Writes a string to the socket.
void memcpy(void *dest, size_t destSize, const void *src, size_t copyCount) noexcept
An OS and compiler independent version of "memcpy".
Definition: os.cpp:358



Page generated by Doxygen 1.8.14 for MRPT 1.9.9 Git: abb8b1a1e Fri Oct 18 14:19:12 2019 +0200 at vie oct 18 14:20:13 CEST 2019