MRPT  1.9.9
net_utils.cpp
Go to the documentation of this file.
1 /* +------------------------------------------------------------------------+
2  | Mobile Robot Programming Toolkit (MRPT) |
3  | https://www.mrpt.org/ |
4  | |
5  | Copyright (c) 2005-2019, Individual contributors, see AUTHORS file |
6  | See: https://www.mrpt.org/Authors - All rights reserved. |
7  | Released under BSD License. See: https://www.mrpt.org/License |
8  +------------------------------------------------------------------------+ */
9 
10 #include "comms-precomp.h" // Precompiled headers
11 
14 #include <mrpt/comms/net_utils.h>
15 #include <mrpt/core/exceptions.h>
16 #include <mrpt/core/format.h>
17 #include <mrpt/system/CTicTac.h>
19 
20 #include <cstdio>
21 #include <cstring>
22 #include <future>
23 #include <thread>
24 
25 #if defined(MRPT_OS_LINUX) || defined(__APPLE__)
26 #define INVALID_SOCKET (-1)
27 #include <arpa/inet.h>
28 #include <fcntl.h>
29 #include <netdb.h>
30 #include <netinet/in.h>
31 #include <sys/ioctl.h>
32 #include <sys/socket.h>
33 #include <sys/types.h>
34 #include <unistd.h>
35 #include <cerrno>
36 #endif
37 
38 #ifdef _WIN32
39 #include <winsock.h>
40 #endif
41 
42 using namespace mrpt;
43 using namespace mrpt::system;
44 using namespace mrpt::comms;
45 using namespace mrpt::comms::net;
46 using namespace std;
47 
48 /*---------------------------------------------------------------
49  http_get
50  ---------------------------------------------------------------*/
52  const string& url, string& out_content, string& out_errormsg, int port,
53  const string& auth_user, const string& auth_pass,
54  int* out_http_responsecode,
55  mrpt::system::TParameters<string>* extra_headers,
56  mrpt::system::TParameters<string>* out_headers, int timeout_ms)
57 {
58  std::vector<uint8_t> data;
60  url, data, out_errormsg, port, auth_user, auth_pass,
61  out_http_responsecode, extra_headers, out_headers, timeout_ms);
62 
63  out_content.resize(data.size());
64  if (!data.empty()) ::memcpy(&out_content[0], &data[0], data.size());
65 
66  return ret;
67 }
68 
70  const string& http_method, const string& http_send_content,
71  const string& url, std::vector<uint8_t>& out_content, string& out_errormsg,
72  int port, const string& auth_user, const string& auth_pass,
73  int* out_http_responsecode,
74  mrpt::system::TParameters<string>* extra_headers,
75  mrpt::system::TParameters<string>* out_headers, int timeout_ms)
76 {
77  // Reset output data:
78  out_content.clear();
79  if (out_http_responsecode) *out_http_responsecode = 0;
80  if (out_headers) out_headers->clear();
81 
82  // URL must be:
83  // http://<SERVER>/<LOCAL_ADDR>
84 
85  if (0 != ::strncmp(url.c_str(), "http://", 7))
86  {
87  out_errormsg = "URL must start with 'http://'";
88  return net::erBadURL;
89  }
90 
91  string server_addr = url.substr(7);
92  string get_object = "/";
93 
94  // Remove from the first "/" on:
95  size_t pos = server_addr.find("/");
96  if (pos == 0)
97  {
98  out_errormsg = "Server name not found in URL";
99  return net::erBadURL;
100  }
101  if (pos != string::npos)
102  {
103  get_object = server_addr.substr(pos);
104  server_addr.resize(pos);
105  }
106 
107  CClientTCPSocket sock;
108 
109  try
110  {
111  // Connect:
112  sock.connect(server_addr, port, timeout_ms);
113  }
114  catch (const std::exception& e)
115  {
116  out_errormsg = e.what();
117  return net::erCouldntConnect;
118  }
119 
120  try
121  {
122  // Set the user-defined headers (we may overwrite them if needed)
123  TParameters<string> headers_to_send;
124  if (extra_headers) headers_to_send = *extra_headers;
125 
126  headers_to_send["Connection"] = "close"; // Don't keep alive
127 
128  if (!headers_to_send.has("User-Agent"))
129  headers_to_send["User-Agent"] = "MRPT Library";
130 
131  // Implement HTTP Basic authentication:
132  // See: http://en.wikipedia.org/wiki/Basic_access_authentication
133  if (!auth_user.empty())
134  {
135  string auth_str = auth_user + string(":") + auth_pass;
136  std::vector<uint8_t> v(auth_str.size());
137  ::memcpy(&v[0], &auth_str[0], auth_str.size());
138 
139  string encoded_str;
140  mrpt::system::encodeBase64(v, encoded_str);
141 
142  headers_to_send["Authorization"] = string("Basic ") + encoded_str;
143  }
144 
145  if (!http_send_content.empty() &&
146  headers_to_send.find("Content-Length") == headers_to_send.end())
147  {
148  headers_to_send["Content-Length"] =
149  mrpt::format("%u", (unsigned int)http_send_content.size());
150  }
151 
152  // Prepare the request string
153  // ---------------------------------
154  string req = format(
155  "%s %s HTTP/1.1\r\n"
156  "Host: %s\r\n",
157  http_method.c_str(), get_object.c_str(), server_addr.c_str());
158 
159  // Other headers:
160  for (auto i = headers_to_send.begin(); i != headers_to_send.end(); ++i)
161  {
162  req += i->first;
163  req += ": ";
164  req += i->second;
165  req += "\r\n";
166  }
167 
168  // End:
169  req += "\r\n";
170 
171  // Any POST data?
172  req += http_send_content;
173 
174  // Send:
175  sock.sendString(req);
176 
177  // Read answer:
178  std::vector<uint8_t> buf;
179  buf.reserve(1 << 14);
180 
181  size_t total_read = 0;
182  bool content_length_read = false;
183  bool all_headers_read = false;
184  size_t content_length = 0;
185  size_t content_offset = 0;
186  int http_code = 0;
188 
189  CTicTac watchdog;
190  watchdog.Tic();
191 
192  while (!content_length_read ||
193  total_read < (content_offset + content_length))
194  {
195  // Read until "Content-Length: XXX \r\n" is read or the whole
196  // message is read,
197  // or an error code is read.
198  size_t to_read_now;
199 
200  if (!content_length_read)
201  {
202  to_read_now = 1500;
203  }
204  else
205  {
206  to_read_now = (content_length + content_offset) - total_read;
207  }
208 
209  // make room for the data to come:
210  buf.resize(total_read + to_read_now + 1);
211 
212  // Read:
213  size_t len =
214  sock.readAsync(&buf[total_read], to_read_now, timeout_ms, 100);
215  if (!len)
216  {
217  //
218  if (!sock.isConnected())
219  {
220  if (all_headers_read) // It seems we're done...
221  break;
222  else
223  {
224  out_errormsg = "Connection to server was lost";
225  return net::erCouldntConnect;
226  }
227  }
228 
229  if (watchdog.Tac() > 1e-3 * timeout_ms)
230  {
231  out_errormsg = "Timeout waiting answer from server";
232  return net::erCouldntConnect;
233  }
234  std::this_thread::sleep_for(10ms);
235  continue;
236  }
237  total_read += len;
238  watchdog.Tic();
239 
240  buf[total_read] = '\0';
241 
242  // do we have a \r\n\r\n ??
243  if (!all_headers_read)
244  {
245  const char* ptr = ::strstr(
246  reinterpret_cast<const char*>(&buf[0]), "\r\n\r\n");
247  if (ptr)
248  {
249  all_headers_read = true;
250  const size_t pos_dblret = ((char*)ptr) - (char*)(&buf[0]);
251 
252  // Process the headers:
253  // ------------------------------
254  if (!::strncmp("HTTP/", (const char*)&buf[0], 5))
255  {
256  http_code = ::atoi((const char*)&buf[9]);
257  }
258  else
259  {
260  // May it be a "SOURCETABLE " answer for NTRIP
261  // protocol??
262  if (!::strncmp(
263  "SOURCETABLE ", (const char*)&buf[0], 12))
264  {
265  http_code = ::atoi((const char*)&buf[12]);
266  }
267  else
268  {
269  out_errormsg =
270  "Server didn't send an HTTP/1.1 answer.";
271  return net::erOtherHTTPError;
272  }
273  }
274 
275  // Check the HTTP code and the content-length:
276  content_offset = pos_dblret + 4;
277 
278  // Do we have a "Content-Length:"??
279  const char* ptr_len = ::strstr(
280  reinterpret_cast<const char*>(&buf[0]),
281  "Content-Length:");
282  if (ptr_len)
283  {
284  content_length = ::atol(ptr_len + 15);
285  content_length_read = true;
286  }
287 
288  // Parse the rest of HTTP headers:
289  {
290  string aux_all_headers;
291  deque<string> lstLines;
292  aux_all_headers.resize(content_offset);
293  ::memcpy(&aux_all_headers[0], &buf[0], content_offset);
294 
296  aux_all_headers, "\r\n", lstLines);
297 
298  for (auto i = lstLines.begin(); i != lstLines.end();
299  ++i)
300  {
301  const size_t p = i->find(":");
302  if (p == string::npos) continue;
303 
304  const string key = i->substr(0, p);
305  const string val = i->substr(p + 2);
306  rx_headers[key] = val;
307  }
308  }
309  }
310  }
311  } // end while
312 
313  if (out_http_responsecode) *out_http_responsecode = http_code;
314  if (out_headers) *out_headers = rx_headers;
315 
316  // Remove the headers from the content:
317  buf.erase(buf.begin(), buf.begin() + content_offset);
318 
319  // Process: "Transfer-Encoding: chunked"
320  if (rx_headers.has("Transfer-Encoding") &&
321  rx_headers["Transfer-Encoding"] == "chunked")
322  {
323  // See: http://en.wikipedia.org/wiki/Chunked_transfer_encoding
324 
325  size_t index = 0;
326  while (index < buf.size())
327  {
328  if (buf[index] == '\r' && buf[index + 1] == '\n')
329  {
330  buf.erase(buf.begin() + index, buf.begin() + index + 2);
331  continue;
332  }
333 
334  const char* pCRLF = ::strstr((const char*)&buf[index], "\r\n");
335  if (!pCRLF) break;
336 
337  const size_t len_substr = ((char*)pCRLF) - (char*)(&buf[index]);
338 
339  string sLen((const char*)&buf[index], len_substr);
340  sLen = string("0x") + sLen;
341 
342  unsigned int lenChunk;
343  int fields = ::sscanf(sLen.c_str(), "%x", &lenChunk);
344  if (!fields) break;
345 
346  // Remove the len of this chunk header from the data:
347  buf.erase(
348  buf.begin() + index, buf.begin() + index + len_substr + 2);
349 
350  index += lenChunk;
351 
352  if (!lenChunk)
353  {
354  buf.resize(index);
355  break;
356  }
357  }
358  }
359 
360  // Set the content output:
361  out_content.swap(buf);
362 
363  if (http_code == 200)
364  {
365  return net::erOk;
366  }
367  else
368  {
369  out_errormsg = format("HTTP error %i", http_code);
370  return net::erOtherHTTPError;
371  }
372  }
373  catch (const std::exception& e)
374  {
375  out_errormsg = e.what();
376  return net::erCouldntConnect;
377  }
378 
379  return net::erOk;
380 }
381 
382 /*---------------------------------------------------------------
383  http_get
384 ---------------------------------------------------------------*/
386  const string& url, std::vector<uint8_t>& out_content, string& out_errormsg,
387  int port, const string& auth_user, const string& auth_pass,
388  int* out_http_responsecode,
389  mrpt::system::TParameters<string>* extra_headers,
390  mrpt::system::TParameters<string>* out_headers, int timeout_ms)
391 {
392  return http_request(
393  "GET", "", url, out_content, out_errormsg, port, auth_user, auth_pass,
394  out_http_responsecode, extra_headers, out_headers, timeout_ms);
395 }
396 
397 /** Resolve a server address by its name, returning its IP address as a string -
398  * This method has a timeout for the maximum time to wait for the DNS server.
399  * For example: server_name="www.google.com" -> out_ip="209.85.227.99"
400  *
401  * \return true on success, false on timeout or other error.
402  */
404  const std::string& server_name, std::string& out_ip,
405  const unsigned int timeout_ms)
406 {
407  // Firstly: If it's a numeric address already, do nothing:
408  if (server_name.find_first_not_of("0123456789. ") == std::string::npos)
409  {
410  // It's a pure IP address:
411  out_ip = server_name;
412  return true;
413  }
414 
415  // Solve DNS --------------
416  // It seems that the only reliable way of *with a timeout* is to launch a
417  // separate thread.
418 
419  std::future<std::string> dns_result_fut =
420  std::async(std::launch::async, [&]() {
421 // Windows-specific stuff:
422 #ifdef _WIN32
423  {
424  // Init the WinSock Library:
425  WORD wVersionRequested = MAKEWORD(2, 0);
426  WSADATA wsaData;
427  if (WSAStartup(wVersionRequested, &wsaData))
428  {
429  std::cerr
430  << "thread_DNS_solver_async: Error calling WSAStartup";
431  return std::string();
432  }
433  }
434 #endif
435 
436  // Do the DNS lookup:
437  std::string dns_result;
438 
439  hostent* he = gethostbyname(server_name.c_str());
440  if (!he)
441  {
442  dns_result.clear(); // empty string -> error.
443  }
444  else
445  {
446  struct in_addr ADDR;
447  ::memcpy(
448  &ADDR, he->h_addr,
449  sizeof(ADDR)); // Was: *((struct in_addr *)he->h_addr);
450  // Convert address to text:
451  dns_result = string(inet_ntoa(ADDR));
452  }
453 
454 #ifdef _WIN32
455  WSACleanup();
456 #endif
457  return dns_result;
458  });
459 
460  auto status =
461  dns_result_fut.wait_for(std::chrono::milliseconds(timeout_ms));
462 
463  if (status == std::future_status::ready)
464  {
465  // Done: Anyway, it can still be an error result:
466  out_ip = dns_result_fut.get();
467  return !out_ip.empty();
468  }
469  else
470  {
471  // Timeout:
472  out_ip.clear();
473 
474  return false;
475  }
476 }
477 
478 /** Returns a description of the last Sockets error */
480 {
481 #ifdef _WIN32
482  const int errnum = WSAGetLastError();
483  char* s = nullptr;
484  FormatMessageA(
485  FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
486  FORMAT_MESSAGE_IGNORE_INSERTS,
487  nullptr, errnum, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&s,
488  0, nullptr);
489  const std::string str = mrpt::format("%s [errno=%d]", s, errnum);
490  LocalFree(s);
491  return str;
492 #else
493  return std::string(strerror(errno));
494 #endif
495 }
496 
498  const std::string& address, const int max_attempts,
499  std::string* output_str /*=NULL*/)
500 {
501  using namespace std;
502 
503  // Format a command string
504  string cmd_str = "ping";
505 
506 // different "count" argument for Windows and *NIX systems
507 #if defined(MRPT_OS_LINUX) || defined(__APPLE__)
508  cmd_str += " -c ";
509 #else
510  cmd_str += " -n ";
511 #endif
512  cmd_str += std::to_string(max_attempts);
513 
514  // Address:
515  cmd_str += " ";
516  cmd_str += address;
517 
518 // Redirection:
519 #if defined(MRPT_OS_LINUX) || defined(__APPLE__)
520  cmd_str += " 2>&1";
521 #endif
522 
523  // Finally exec the command
524  int code = executeCommand(cmd_str, output_str);
525 
526  return (code == 0);
527 }
double Tac() noexcept
Stops the stopwatch.
Definition: CTicTac.cpp:86
iterator begin() noexcept
Definition: TParameters.h:134
void connect(const std::string &remotePartAddress, unsigned short remotePartTCPPort, unsigned int timeout_ms=0)
Establishes a connection with a remote part.
bool isConnected()
Returns true if this objects represents a successfully connected socket.
A high-performance stopwatch, with typical resolution of nanoseconds.
std::string getLastSocketErrorStr()
Returns a description of the last Sockets error.
Definition: net_utils.cpp:479
iterator end() noexcept
Definition: TParameters.h:136
STL namespace.
GLdouble s
Definition: glext.h:3682
GLenum GLsizei len
Definition: glext.h:4756
ERRORCODE_HTTP http_get(const string &url, std::vector< uint8_t > &out_content, string &out_errormsg, int port=80, const string &auth_user=string(), const string &auth_pass=string(), int *out_http_responsecode=nullptr, mrpt::system::TParameters< string > *extra_headers=nullptr, mrpt::system::TParameters< string > *out_headers=nullptr, int timeout_ms=1000)
Perform an HTTP GET operation (version for retrieving the data as a std::vector<uint8_t>) ...
Definition: net_utils.cpp:385
void tokenize(const std::string &inString, const std::string &inDelimiters, OUT_CONTAINER &outTokens, bool skipBlankTokens=true) noexcept
Tokenizes a string according to a set of delimiting characters.
bool Ping(const std::string &address, const int max_attempts, std::string *output_str=nullptr)
Ping an IP address.
Definition: net_utils.cpp:497
ERRORCODE_HTTP
Possible returns from a HTTP request.
Definition: net_utils.h:31
bool has(const std::string &s) const
Definition: TParameters.h:69
bool DNS_resolve_async(const std::string &server_name, std::string &out_ip, const unsigned int timeout_ms=3000)
Resolve a server address by its name, returning its IP address as a string - This method has a timeou...
Definition: net_utils.cpp:403
GLuint index
Definition: glext.h:4068
int val
Definition: mrpt_jpeglib.h:957
GLsizei const GLchar ** string
Definition: glext.h:4116
iterator find(const std::string &key)
Definition: TParameters.h:132
int executeCommand(const std::string &command, std::string *output=nullptr, const std::string &mode="r")
Execute Generic Shell Command.
Definition: os.cpp:662
const GLdouble * v
Definition: glext.h:3684
This is the global namespace for all Mobile Robot Programming Toolkit (MRPT) libraries.
Definition: inftrees.h:28
void encodeBase64(const std::vector< uint8_t > &inputData, std::string &outString)
Encode a sequence of bytes as a string in base-64.
Definition: base64.cpp:29
std::string format(const char *fmt,...) MRPT_printf_format_check(1
A std::string version of C sprintf.
Definition: format.cpp:16
A TCP socket that can be connected to a TCP server, implementing MRPT&#39;s CStream interface for passing...
_u8 status
Definition: rplidar_cmd.h:19
ERRORCODE_HTTP http_request(const string &http_method, const string &http_send_content, const string &url, std::vector< uint8_t > &out_content, string &out_errormsg, int port=80, const string &auth_user=string(), const string &auth_pass=string(), int *out_http_responsecode=nullptr, mrpt::system::TParameters< string > *extra_headers=nullptr, mrpt::system::TParameters< string > *out_headers=nullptr, int timeout_ms=1000)
Generic function for HTTP GET & POST methods.
Definition: net_utils.cpp:69
A set of useful routines for networking.
Definition: net_utils.h:23
GLuint address
Definition: glext.h:7060
std::string std::string to_string(T v)
Just like std::to_string(), but with an overloaded version for std::string arguments.
Definition: format.h:30
Serial and networking devices and utilities.
void Tic() noexcept
Starts the stopwatch.
Definition: CTicTac.cpp:75
size_t readAsync(void *Buffer, const size_t Count, const int timeoutStart_ms=-1, const int timeoutBetween_ms=-1)
A method for reading from the socket with an optional timeout.
For usage when passing a dynamic number of (numeric) arguments to a function, by name.
Definition: TParameters.h:54
GLsizei GLsizei GLenum GLenum const GLvoid * data
Definition: glext.h:3550
GLfloat GLfloat p
Definition: glext.h:6398
void sendString(const std::string &str)
Writes a string to the socket.
void memcpy(void *dest, size_t destSize, const void *src, size_t copyCount) noexcept
An OS and compiler independent version of "memcpy".
Definition: os.cpp:358



Page generated by Doxygen 1.8.14 for MRPT 1.9.9 Git: 8fe78517f Sun Jul 14 19:43:28 2019 +0200 at lun oct 28 02:10:00 CET 2019