libzypp  17.31.8
request.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 ----------------------------------------------------------------------*/
13 #include <zypp-core/zyppng/base/EventDispatcher>
14 #include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
15 #include <zypp-core/zyppng/core/String>
17 #include <zypp-curl/CurlConfig>
18 #include <zypp-curl/auth/CurlAuthData>
19 #include <zypp-media/MediaConfig>
20 #include <zypp-core/base/String.h>
21 #include <zypp-core/base/StringV.h>
22 #include <zypp-core/Pathname.h>
23 #include <curl/curl.h>
24 #include <stdio.h>
25 #include <fcntl.h>
26 #include <sstream>
27 #include <utility>
28 
29 #include <iostream>
30 #include <boost/variant.hpp>
31 #include <boost/variant/polymorphic_get.hpp>
32 
33 
34 namespace zyppng {
35 
36  void dump(const char *text,
37  FILE *stream, unsigned char *ptr, size_t size)
38  {
39  size_t i;
40  size_t c;
41  unsigned int width=0x10;
42 
43  DBG << zypp::str::form ( "%s, %10.10ld bytes (0x%8.8lx)\n", text, (long)size, (long)size );
44 
45  for(i=0; i<size; i+= width) {
46 
47  DBG << zypp::str::form ( "%4.4lx: ", (long)i );
48 
49  /* show hex to the left */
50  for(c = 0; c < width; c++) {
51  if(i+c < size)
52  DBG << zypp::str::form("%02x ", ptr[i+c]);
53  else
54  DBG << (" ");
55  }
56 
57  /* show data on the right */
58  for(c = 0; (c < width) && (i+c < size); c++) {
59  char x = (ptr[i+c] >= 0x20 && ptr[i+c] < 0x80) ? ptr[i+c] : '.';
60  DBG << x;
61  }
62 
63  DBG << std::endl;
64  }
65  }
66 
67  int log_curl_all(CURL *handle, curl_infotype type,
68  char *data, size_t size,
69  void *userp)
70  {
71  const char *text;
72 
73  long maxlvl = userp == nullptr ? 3 : *((long *)userp);
74 
75  (void)handle; /* prevent compiler warning */
76  (void)userp;
77 
78  switch (type) {
79  case CURLINFO_TEXT:
80  DBG << handle << " " << "== Info: ";
81  DBG << handle << " " << std::string( data, size ) << std::endl;
82 
83  default: /* in case a new one is introduced to shock us */
84  return 0;
85 
86  case CURLINFO_HEADER_OUT:
87  text = "=> Send header: ";
88  DBG << handle << " " << std::string( data, size ) << std::endl;
89  return 0;
90  case CURLINFO_DATA_OUT:
91  text = "=> Send data";
92  break;
93  case CURLINFO_SSL_DATA_OUT:
94  text = "=> Send SSL data";
95  break;
96  case CURLINFO_HEADER_IN:
97  text = "<= Recv header: ";
98  DBG << handle << " " << std::string( data, size ) << std::endl;
99  return 0;
100  case CURLINFO_DATA_IN:
101  text = "<= Recv data";
102  break;
103  case CURLINFO_SSL_DATA_IN:
104  text = "<= Recv SSL data";
105  break;
106  }
107 
108  if ( maxlvl > 4 )
109  dump(text, stderr, (unsigned char *)data, size);
110  else
111  DBG << handle << " " << " " << size << " bytes " << std::endl;
112  return 0;
113  }
114 
115  namespace {
116  static size_t nwr_headerCallback ( char *ptr, size_t size, size_t nmemb, void *userdata ) {
117  if ( !userdata )
118  return 0;
119 
120  NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( userdata );
121  return that->headerCallback( ptr, size, nmemb );
122  }
123  static size_t nwr_writeCallback ( char *ptr, size_t size, size_t nmemb, void *userdata ) {
124  if ( !userdata )
125  return 0;
126 
127  NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( userdata );
128  return that->writeCallback( ptr, size, nmemb );
129  }
130 
131  //helper for std::visit
132  template<class T> struct always_false : std::false_type {};
133  }
134 
135  std::vector<char> peek_data_fd( FILE *fd, off_t offset, size_t count )
136  {
137  if ( !fd )
138  return {};
139 
140  fflush( fd );
141 
142  std::vector<char> data( count + 1 , '\0' );
143 
144  ssize_t l = -1;
145  while ((l = pread( fileno( fd ), data.data(), count, offset ) ) == -1 && errno == EINTR)
146  ;
147  if (l == -1)
148  return {};
149 
150  return data;
151  }
152 
153  NetworkRequest::Range NetworkRequest::Range::make(size_t start, size_t len, zyppng::NetworkRequest::DigestPtr &&digest, zyppng::NetworkRequest::CheckSumBytes &&expectedChkSum, std::any &&userData, std::optional<size_t> digestCompareLen, std::optional<size_t> dataBlockPadding )
154  {
155  return NetworkRequest::Range {
156  .start = start,
157  .len = len,
158  .bytesWritten = 0,
159  ._digest = std::move( digest ),
160  ._checksum = std::move( expectedChkSum ),
161  ._relevantDigestLen = std::move( digestCompareLen ),
162  ._chksumPad = std::move( dataBlockPadding ),
163  .userData = std::move( userData ),
164  ._rangeState = State::Pending
165  };
166  }
167 
169  : _outFile( std::move(prevState._outFile) )
170  , _downloaded( prevState._downloaded )
171  , _rangeAttemptIdx( prevState._rangeAttemptIdx )
172  { }
173 
175  : _requireStatusPartial( prevState._requireStatusPartial )
176  { }
177 
179  : _outFile( std::move(prevState._outFile) )
180  , _requireStatusPartial( true )
181  , _downloaded( prevState._downloaded )
182  , _rangeAttemptIdx( prevState._rangeAttemptIdx )
183  { }
184 
186  : BasePrivate(p)
187  , _url ( std::move(url) )
188  , _targetFile ( std::move( targetFile) )
189  , _fMode ( std::move(fMode) )
190  , _headers( std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( nullptr, &curl_slist_free_all ) )
191  { }
192 
194  {
195  if ( _easyHandle ) {
196  //clean up for now, later we might reuse handles
197  curl_easy_cleanup( _easyHandle );
198  //reset in request but make sure the request was not enqueued again and got a new handle
199  _easyHandle = nullptr;
200  }
201  }
202 
203  bool NetworkRequestPrivate::initialize( std::string &errBuf )
204  {
205  reset();
206 
207  if ( _easyHandle )
208  //will reset to defaults but keep live connections, session ID and DNS caches
209  curl_easy_reset( _easyHandle );
210  else
211  _easyHandle = curl_easy_init();
212  return setupHandle ( errBuf );
213  }
214 
215  bool NetworkRequestPrivate::setupHandle( std::string &errBuf )
216  {
217  curl_easy_setopt( _easyHandle, CURLOPT_ERRORBUFFER, this->_errorBuf.data() );
218 
219  const std::string urlScheme = _url.getScheme();
220  if ( urlScheme == "http" || urlScheme == "https" )
222 
223  try {
224 
225  setCurlOption( CURLOPT_PRIVATE, this );
226  setCurlOption( CURLOPT_XFERINFOFUNCTION, NetworkRequestPrivate::curlProgressCallback );
227  setCurlOption( CURLOPT_XFERINFODATA, this );
228  setCurlOption( CURLOPT_NOPROGRESS, 0L);
229  setCurlOption( CURLOPT_FAILONERROR, 1L);
230  setCurlOption( CURLOPT_NOSIGNAL, 1L);
231 
232  std::string urlBuffer( _url.asString() );
233  setCurlOption( CURLOPT_URL, urlBuffer.c_str() );
234 
235  setCurlOption( CURLOPT_WRITEFUNCTION, nwr_writeCallback );
236  setCurlOption( CURLOPT_WRITEDATA, this );
237 
239  setCurlOption( CURLOPT_CONNECT_ONLY, 1L );
240  setCurlOption( CURLOPT_FRESH_CONNECT, 1L );
241  }
243  // instead of returning no data with NOBODY, we return
244  // little data, that works with broken servers, and
245  // works for ftp as well, because retrieving only headers
246  // ftp will return always OK code ?
247  // See http://curl.haxx.se/docs/knownbugs.html #58
249  setCurlOption( CURLOPT_NOBODY, 1L );
250  else
251  setCurlOption( CURLOPT_RANGE, "0-1" );
252  }
253 
255  if ( _requestedRanges.size() ) {
256  if ( ! prepareNextRangeBatch ( errBuf ))
257  return false;
258  } else {
259  std::visit( [&]( auto &arg ){
260  using T = std::decay_t<decltype(arg)>;
261  if constexpr ( std::is_same_v<T, pending_t> ) {
262  arg._requireStatusPartial = false;
263  } else {
264  DBG << _easyHandle << " " << "NetworkRequestPrivate::setupHandle called in unexpected state" << std::endl;
265  }
266  }, _runningMode );
268  _requestedRanges.back()._rangeState = NetworkRequest::State::Running;
269  }
270  }
271 
272  //make a local copy of the settings, so headers are not added multiple times
273  TransferSettings locSet = _settings;
274 
275  if ( _dispatcher ) {
276  locSet.setUserAgentString( _dispatcher->agentString().c_str() );
277 
278  // add custom headers as configured (bsc#955801)
279  const auto &cHeaders = _dispatcher->hostSpecificHeaders();
280  if ( auto i = cHeaders.find(_url.getHost()); i != cHeaders.end() ) {
281  for ( const auto &[key, value] : i->second ) {
283  "%s: %s", key.c_str(), value.c_str() )
284  ));
285  }
286  }
287  }
288 
289  locSet.addHeader("Pragma:");
290 
291  locSet.setTimeout( zypp::MediaConfig::instance().download_transfer_timeout() );
293 
294  {
295  char *ptr = getenv("ZYPP_MEDIA_CURL_DEBUG");
296  _curlDebug = (ptr && *ptr) ? zypp::str::strtonum<long>( ptr) : 0L;
297  if( _curlDebug > 0)
298  {
299  setCurlOption( CURLOPT_VERBOSE, 1L);
300  if ( _curlDebug >= 3 )
301  setCurlOption( CURLOPT_DEBUGFUNCTION, log_curl_all );
302  else
303  setCurlOption( CURLOPT_DEBUGFUNCTION, ::internal::log_curl);
304  setCurlOption( CURLOPT_DEBUGDATA, &_curlDebug);
305  }
306  }
307 
310  {
311  case 4: setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 ); break;
312  case 6: setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6 ); break;
313  default: break;
314  }
315 
316  setCurlOption( CURLOPT_HEADERFUNCTION, &nwr_headerCallback );
317  setCurlOption( CURLOPT_HEADERDATA, this );
318 
322  setCurlOption( CURLOPT_CONNECTTIMEOUT, locSet.connectTimeout() );
323  // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
324  // just in case curl does not trigger its progress callback frequently
325  // enough.
326  if ( locSet.timeout() )
327  {
328  setCurlOption( CURLOPT_TIMEOUT, 3600L );
329  }
330 
331  if ( urlScheme == "https" )
332  {
333 #if CURLVERSION_AT_LEAST(7,19,4)
334  // restrict following of redirections from https to https only
335  if ( _url.getHost() == "download.opensuse.org" )
336  setCurlOption( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS );
337  else
338  setCurlOption( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
339 #endif
340 
341  if( locSet.verifyPeerEnabled() ||
342  locSet.verifyHostEnabled() )
343  {
344  setCurlOption(CURLOPT_CAPATH, locSet.certificateAuthoritiesPath().c_str());
345  }
346 
347  if( ! locSet.clientCertificatePath().empty() )
348  {
349  setCurlOption(CURLOPT_SSLCERT, locSet.clientCertificatePath().c_str());
350  }
351  if( ! locSet.clientKeyPath().empty() )
352  {
353  setCurlOption(CURLOPT_SSLKEY, locSet.clientKeyPath().c_str());
354  }
355 
356 #ifdef CURLSSLOPT_ALLOW_BEAST
357  // see bnc#779177
358  setCurlOption( CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
359 #endif
360  setCurlOption(CURLOPT_SSL_VERIFYPEER, locSet.verifyPeerEnabled() ? 1L : 0L);
361  setCurlOption(CURLOPT_SSL_VERIFYHOST, locSet.verifyHostEnabled() ? 2L : 0L);
362  // bnc#903405 - POODLE: libzypp should only talk TLS
363  setCurlOption(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
364  }
365 
366  // follow any Location: header that the server sends as part of
367  // an HTTP header (#113275)
368  setCurlOption( CURLOPT_FOLLOWLOCATION, 1L);
369  // 3 redirects seem to be too few in some cases (bnc #465532)
370  setCurlOption( CURLOPT_MAXREDIRS, 6L );
371 
372  //set the user agent
373  setCurlOption(CURLOPT_USERAGENT, locSet.userAgentString().c_str() );
374 
375 
376  /*---------------------------------------------------------------*
377  CURLOPT_USERPWD: [user name]:[password]
378  Url::username/password -> CURLOPT_USERPWD
379  If not provided, anonymous FTP identification
380  *---------------------------------------------------------------*/
381  if ( locSet.userPassword().size() )
382  {
383  setCurlOption(CURLOPT_USERPWD, locSet.userPassword().c_str());
384  std::string use_auth = _settings.authType();
385  if (use_auth.empty())
386  use_auth = "digest,basic"; // our default
387  long auth = zypp::media::CurlAuthData::auth_type_str2long(use_auth);
388  if( auth != CURLAUTH_NONE)
389  {
390  DBG << _easyHandle << " " << "Enabling HTTP authentication methods: " << use_auth
391  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
392  setCurlOption(CURLOPT_HTTPAUTH, auth);
393  }
394  }
395 
396  if ( locSet.proxyEnabled() && ! locSet.proxy().empty() )
397  {
398  DBG << _easyHandle << " " << "Proxy: '" << locSet.proxy() << "'" << std::endl;
399  setCurlOption(CURLOPT_PROXY, locSet.proxy().c_str());
400  setCurlOption(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
401 
402  /*---------------------------------------------------------------*
403  * CURLOPT_PROXYUSERPWD: [user name]:[password]
404  *
405  * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
406  * If not provided, $HOME/.curlrc is evaluated
407  *---------------------------------------------------------------*/
408 
409  std::string proxyuserpwd = locSet.proxyUserPassword();
410 
411  if ( proxyuserpwd.empty() )
412  {
413  zypp::media::CurlConfig curlconf;
414  zypp::media::CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
415  if ( curlconf.proxyuserpwd.empty() )
416  DBG << _easyHandle << " " << "Proxy: ~/.curlrc does not contain the proxy-user option" << std::endl;
417  else
418  {
419  proxyuserpwd = curlconf.proxyuserpwd;
420  DBG << _easyHandle << " " << "Proxy: using proxy-user from ~/.curlrc" << std::endl;
421  }
422  }
423  else
424  {
425  DBG << _easyHandle << " " << _easyHandle << " " << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << std::endl;
426  }
427 
428  if ( ! proxyuserpwd.empty() )
429  {
430  setCurlOption(CURLOPT_PROXYUSERPWD, ::internal::curlUnEscape( proxyuserpwd ).c_str());
431  }
432  }
433 #if CURLVERSION_AT_LEAST(7,19,4)
434  else if ( locSet.proxy() == EXPLICITLY_NO_PROXY )
435  {
436  // Explicitly disabled in URL (see fillSettingsFromUrl()).
437  // This should also prevent libcurl from looking into the environment.
438  DBG << _easyHandle << " " << "Proxy: explicitly NOPROXY" << std::endl;
439  setCurlOption(CURLOPT_NOPROXY, "*");
440  }
441 
442 #endif
443  else
444  {
445  DBG << _easyHandle << " " << "Proxy: not explicitly set" << std::endl;
446  DBG << _easyHandle << " " << "Proxy: libcurl may look into the environment" << std::endl;
447  }
448 
450  if ( locSet.minDownloadSpeed() != 0 )
451  {
452  setCurlOption(CURLOPT_LOW_SPEED_LIMIT, locSet.minDownloadSpeed());
453  // default to 10 seconds at low speed
454  setCurlOption(CURLOPT_LOW_SPEED_TIME, 60L);
455  }
456 
457 #if CURLVERSION_AT_LEAST(7,15,5)
458  if ( locSet.maxDownloadSpeed() != 0 )
459  setCurlOption(CURLOPT_MAX_RECV_SPEED_LARGE, locSet.maxDownloadSpeed());
460 #endif
461 
462  if ( zypp::str::strToBool( _url.getQueryParam( "cookies" ), true ) )
463  setCurlOption( CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
464  else
465  MIL << _easyHandle << " " << "No cookies requested" << std::endl;
466  setCurlOption(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
467 
468 #if CURLVERSION_AT_LEAST(7,18,0)
469  // bnc #306272
470  setCurlOption(CURLOPT_PROXY_TRANSFER_MODE, 1L );
471 #endif
472 
473  // append settings custom headers to curl
474  for ( const auto &header : locSet.headers() ) {
475  if ( !z_func()->addRequestHeader( header.c_str() ) )
477  }
478 
479  if ( _headers )
480  setCurlOption( CURLOPT_HTTPHEADER, _headers.get() );
481 
482  return true;
483 
484  } catch ( const zypp::Exception &excp ) {
485  ZYPP_CAUGHT(excp);
486  errBuf = excp.asString();
487  }
488  return false;
489  }
490 
492  {
493  auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
494  if ( !rmode ) {
495  DBG << _easyHandle << "Can only create output file in running mode" << std::endl;
496  return false;
497  }
498  // if we have no open file create or open it
499  if ( !rmode->_outFile ) {
500  std::string openMode = "w+b";
502  openMode = "r+b";
503 
504  rmode->_outFile = fopen( _targetFile.asString().c_str() , openMode.c_str() );
505 
506  //if the file does not exist create a new one
507  if ( !rmode->_outFile && _fMode == NetworkRequest::WriteShared ) {
508  rmode->_outFile = fopen( _targetFile.asString().c_str() , "w+b" );
509  }
510 
511  if ( !rmode->_outFile ) {
513  ,zypp::str::Format("Unable to open target file (%1%). Errno: (%2%:%3%)") % _targetFile.asString() % errno % strerr_cxx() );
514  return false;
515  }
516  }
517 
518  return true;
519  }
520 
522  {
523  // We can recover from RangeFail errors if we have more batch sizes to try
524  auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
525  if ( rmode->_cachedResult && rmode->_cachedResult->type() == NetworkRequestError::RangeFail )
526  return ( rmode->_rangeAttemptIdx + 1 < sizeof( _rangeAttempt ) ) && hasMoreWork();
527  return false;
528  }
529 
530  bool NetworkRequestPrivate::prepareToContinue( std::string &errBuf )
531  {
532  auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
533 
534  if ( hasMoreWork() ) {
535  // go to the next range batch level if we are restarted due to a failed range request
536  if ( rmode->_cachedResult && rmode->_cachedResult->type() == NetworkRequestError::RangeFail ) {
537  if ( rmode->_rangeAttemptIdx + 1 >= sizeof( _rangeAttempt ) ) {
538  errBuf = "No more range batch sizes available";
539  return false;
540  }
541  rmode->_rangeAttemptIdx++;
542  }
543 
544  _runningMode = prepareNextRangeBatch_t( std::move(std::get<running_t>( _runningMode )) );
545 
546  // we reset the handle to default values. We do this to not run into
547  // "transfer closed with outstanding read data remaining" error CURL sometimes returns when
548  // we cancel a connection because of a range error to request a smaller batch.
549  // The error will still happen but much less frequently than without resetting the handle.
550  //
551  // Note: Even creating a new handle will NOT fix the issue
552  curl_easy_reset( _easyHandle );
553  if ( !setupHandle (errBuf) )
554  return false;
555  return true;
556  }
557  errBuf = "Request has no more work";
558  return false;
559 
560  }
561 
563  {
564  if ( _requestedRanges.size() == 0 ) {
565  errBuf = "Calling the prepareNextRangeBatch function without a range to download is not supported.";
566  return false;
567  }
568 
569  std::string rangeDesc;
570  uint rangesAdded = 0;
571  if ( _requestedRanges.size() > 1 && _protocolMode != ProtocolMode::HTTP ) {
572  errBuf = "Using more than one range is not supported with protocols other than HTTP/HTTPS";
573  return false;
574  }
575 
576  // check if we have one big range convering the whole file
577  if ( _requestedRanges.size() == 1 && _requestedRanges.front().start == 0 && _requestedRanges.front().len == 0 ) {
578  if ( !std::holds_alternative<pending_t>( _runningMode ) ) {
579  errBuf = zypp::str::Str() << "Unexpected state when calling prepareNextRangeBatch " << _runningMode.index ();
580  return false;
581  }
582 
583  _requestedRanges[0]._rangeState = NetworkRequest::Running;
584  std::get<pending_t>( _runningMode )._requireStatusPartial = false;
585 
586  } else {
587  std::sort( _requestedRanges.begin(), _requestedRanges.end(), []( const auto &elem1, const auto &elem2 ){
588  return ( elem1.start < elem2.start );
589  });
590 
591  if ( std::holds_alternative<pending_t>( _runningMode ) )
592  std::get<pending_t>( _runningMode )._requireStatusPartial = true;
593 
594  auto maxRanges = _rangeAttempt[0];
595  if ( std::holds_alternative<prepareNextRangeBatch_t>( _runningMode ) )
596  maxRanges = _rangeAttempt[std::get<prepareNextRangeBatch_t>( _runningMode )._rangeAttemptIdx];
597 
598  // helper function to build up the request string for the range
599  auto addRangeString = [ &rangeDesc, &rangesAdded ]( const std::pair<size_t, size_t> &range ) {
600  std::string rangeD = zypp::str::form("%llu-", static_cast<unsigned long long>( range.first ) );
601  if( range.second > 0 )
602  rangeD.append( zypp::str::form( "%llu", static_cast<unsigned long long>( range.second ) ) );
603 
604  if ( rangeDesc.size() )
605  rangeDesc.append(",").append( rangeD );
606  else
607  rangeDesc = std::move( rangeD );
608 
609  rangesAdded++;
610  };
611 
612  std::optional<std::pair<size_t, size_t>> currentZippedRange;
613  bool closedRange = true;
614  for ( auto &range : _requestedRanges ) {
615 
616  if ( range._rangeState != NetworkRequest::Pending )
617  continue;
618 
619  //reset the download results
620  range.bytesWritten = 0;
621 
622  //when we have a open range in the list of ranges we will get from start of range to end of file,
623  //all following ranges would never be marked as valid, so we have to fail early
624  if ( !closedRange ) {
625  errBuf = "It is not supported to request more ranges after a open range.";
626  return false;
627  }
628 
629  const auto rangeEnd = range.len > 0 ? range.start + range.len - 1 : 0;
630  closedRange = (rangeEnd > 0);
631 
632  // remember this range was already requested
633  range._rangeState = NetworkRequest::Running;
634  range.bytesWritten = 0;
635  if ( range._digest )
636  range._digest->reset();
637 
638  // we try to compress the requested ranges into as big chunks as possible for the request,
639  // when receiving we still track the original ranges so we can collect and test their checksums
640  if ( !currentZippedRange ) {
641  currentZippedRange = std::make_pair( range.start, rangeEnd );
642  } else {
643  //range is directly consecutive to the previous range
644  if ( currentZippedRange->second + 1 == range.start ) {
645  currentZippedRange->second = rangeEnd;
646  } else {
647  //this range does not directly follow the previous one, we build the string and start a new one
648  addRangeString( *currentZippedRange );
649  currentZippedRange = std::make_pair( range.start, rangeEnd );
650  }
651  }
652 
653  if ( rangesAdded >= maxRanges ) {
654  MIL << _easyHandle << " " << "Reached max nr of ranges (" << maxRanges << "), batching the request to not break the server" << std::endl;
655  break;
656  }
657  }
658 
659  // add the last range too
660  if ( currentZippedRange )
661  addRangeString( *currentZippedRange );
662 
663  MIL << _easyHandle << " " << "Requesting Ranges: " << rangeDesc << std::endl;
664 
665  setCurlOption( CURLOPT_RANGE, rangeDesc.c_str() );
666  }
667 
668  return true;
669  }
670 
672  {
673  // check if we have ranges that have never been requested
674  return std::any_of( _requestedRanges.begin(), _requestedRanges.end(), []( const auto &range ){ return range._rangeState == NetworkRequest::Pending; });
675  }
676 
678  {
679  bool isRangeContinuation = std::holds_alternative<prepareNextRangeBatch_t>( _runningMode );
680  if ( isRangeContinuation ) {
681  MIL << _easyHandle << " " << "Continuing a previously started range batch." << std::endl;
682  _runningMode = running_t( std::move(std::get<prepareNextRangeBatch_t>( _runningMode )) );
683  } else {
684  auto mode = running_t( std::move(std::get<pending_t>( _runningMode )) );
685  if ( _requestedRanges.size() == 1 && _requestedRanges.front().start == 0 && _requestedRanges.front().len == 0 )
686  mode._currentRange = 0;
687 
688  _runningMode = std::move(mode);
689  }
690 
691  auto &m = std::get<running_t>( _runningMode );
692 
693  if ( m._activityTimer ) {
694  DBG_MEDIA << _easyHandle << " Setting activity timeout to: " << _settings.timeout() << std::endl;
695  m._activityTimer->connect( &Timer::sigExpired, *this, &NetworkRequestPrivate::onActivityTimeout );
696  m._activityTimer->start( static_cast<uint64_t>( _settings.timeout() * 1000 ) );
697  }
698 
699  if ( !isRangeContinuation )
700  _sigStarted.emit( *z_func() );
701  }
702 
704  {
705  if ( std::holds_alternative<running_t>(_runningMode) ) {
706  auto &rmode = std::get<running_t>( _runningMode );
707  // if we still have a current range set it valid by checking the checksum
708  if ( rmode._currentRange >= 0 ) {
709  auto &currR = _requestedRanges[rmode._currentRange];
710  rmode._currentRange = -1;
711  validateRange( currR );
712  }
713  }
714  }
715 
717  {
718 
719  finished_t resState;
720  resState._result = std::move(err);
721 
722  if ( std::holds_alternative<running_t>(_runningMode) ) {
723 
724  auto &rmode = std::get<running_t>( _runningMode );
725  rmode._outFile.reset();
726  resState._downloaded = rmode._downloaded;
727  resState._contentLenght = rmode._contentLenght;
728 
730  //we have a successful download lets see if we got everything we needed
731  for ( const auto &r : _requestedRanges ) {
732  if ( r._rangeState != NetworkRequest::Finished ) {
733  if ( r.len > 0 && r.bytesWritten != r.len )
734  resState._result = NetworkRequestErrorPrivate::customError( NetworkRequestError::MissingData, (zypp::str::Format("Did not receive all requested data from the server ( off: %1%, req: %2%, recv: %3% ).") % r.start % r.len % r.bytesWritten ) );
735  else if ( r._digest && r._checksum.size() && ! checkIfRangeChkSumIsValid(r) ) {
736  resState._result = NetworkRequestErrorPrivate::customError( NetworkRequestError::InvalidChecksum, (zypp::str::Format("Invalid checksum %1%, expected checksum %2%") % r._digest->digest() % zypp::Digest::digestVectorToString( r._checksum ) ) );
737  } else {
739  }
740  //we only report the first error
741  break;
742  }
743  }
744  }
745  }
746 
747  _runningMode = std::move( resState );
748  _sigFinished.emit( *z_func(), std::get<finished_t>(_runningMode)._result );
749  }
750 
752  {
754  _headers.reset( nullptr );
755  _errorBuf.fill( 0 );
757  std::for_each( _requestedRanges.begin (), _requestedRanges.end(), []( auto &range ) {
758  range._rangeState = NetworkRequest::Pending;
759  });
760  }
761 
763  {
764  auto &m = std::get<running_t>( _runningMode );
765 
766  MIL_MEDIA << _easyHandle << " Request timeout interval: " << t.interval()<< " remaining: " << t.remaining() << std::endl;
767  std::map<std::string, boost::any> extraInfo;
768  extraInfo.insert( {"requestUrl", _url } );
769  extraInfo.insert( {"filepath", _targetFile } );
770  _dispatcher->cancel( *z_func(), NetworkRequestErrorPrivate::customError( NetworkRequestError::Timeout, "Download timed out", std::move(extraInfo) ) );
771  }
772 
774  {
775  if ( rng._digest && rng._checksum.size() ) {
776  auto bytesHashed = rng._digest->bytesHashed ();
777  if ( rng._chksumPad && *rng._chksumPad > bytesHashed ) {
778  MIL_MEDIA << _easyHandle << " " << "Padding the digest to required block size" << std::endl;
779  zypp::ByteArray padding( *rng._chksumPad - bytesHashed, '\0' );
780  rng._digest->update( padding.data(), padding.size() );
781  }
782  auto digVec = rng._digest->digestVector();
783  if ( rng._relevantDigestLen ) {
784  digVec.resize( *rng._relevantDigestLen );
785  }
786  return ( digVec == rng._checksum );
787  }
788 
789  // no checksum required
790  return true;
791  }
792 
794  {
795  if ( rng._digest && rng._checksum.size() ) {
796  if ( ( rng.len == 0 || rng.bytesWritten == rng.len ) && checkIfRangeChkSumIsValid(rng) )
798  else
800  } else {
801  if ( rng.len == 0 ? true : rng.bytesWritten == rng.len )
803  else
805  }
806  }
807 
808  bool NetworkRequestPrivate::parseContentRangeHeader(const std::string_view &line, size_t &start, size_t &len )
809  { //content-range: bytes 10485760-19147879/19147880
810  static const zypp::str::regex regex("^Content-Range:[[:space:]]+bytes[[:space:]]+([0-9]+)-([0-9]+)\\/([0-9]+)$", zypp::str::regex::rxdefault | zypp::str::regex::icase );
811 
812  zypp::str::smatch what;
813  if( !zypp::str::regex_match( std::string(line), what, regex ) || what.size() != 4 ) {
814  DBG << _easyHandle << " " << "Invalid Content-Range Header format: '" << std::string(line) << std::endl;
815  return false;
816  }
817 
818  size_t s = zypp::str::strtonum<size_t>( what[1]);
819  size_t e = zypp::str::strtonum<size_t>( what[2]);
820  start = std::move(s);
821  len = ( e - s ) + 1;
822  return true;
823  }
824 
825  bool NetworkRequestPrivate::parseContentTypeMultiRangeHeader(const std::string_view &line, std::string &boundary)
826  {
827  static const zypp::str::regex regex("^Content-Type:[[:space:]]+multipart\\/byteranges;[[:space:]]+boundary=(.*)$", zypp::str::regex::rxdefault | zypp::str::regex::icase );
828 
829  zypp::str::smatch what;
830  if( zypp::str::regex_match( std::string(line), what, regex ) ) {
831  if ( what.size() >= 2 ) {
832  boundary = what[1];
833  return true;
834  }
835  }
836  return false;
837  }
838 
840  {
841  return std::string( _errorBuf.data() );
842  }
843 
845  {
846  if ( std::holds_alternative<running_t>( _runningMode ) ){
847  auto &rmode = std::get<running_t>( _runningMode );
848  if ( rmode._activityTimer && rmode._activityTimer->isRunning() )
849  rmode._activityTimer->start();
850  }
851  }
852 
853  int NetworkRequestPrivate::curlProgressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow )
854  {
855  if ( !clientp )
856  return CURLE_OK;
857  NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( clientp );
858 
859  if ( !std::holds_alternative<running_t>(that->_runningMode) ){
860  DBG << that->_easyHandle << " " << "Curl progress callback was called in invalid state "<< that->z_func()->state() << std::endl;
861  return -1;
862  }
863 
864  auto &rmode = std::get<running_t>( that->_runningMode );
865 
866  //reset the timer
867  that->resetActivityTimer();
868 
869  rmode._isInCallback = true;
870  if ( rmode._lastProgressNow != dlnow ) {
871  rmode._lastProgressNow = dlnow;
872  that->_sigProgress.emit( *that->z_func(), dltotal, dlnow, ultotal, ulnow );
873  }
874  rmode._isInCallback = false;
875 
876  return rmode._cachedResult ? CURLE_ABORTED_BY_CALLBACK : CURLE_OK;
877  }
878 
879  size_t NetworkRequestPrivate::headerCallback(char *ptr, size_t size, size_t nmemb)
880  {
881  //it is valid to call this function with no data to write, just return OK
882  if ( size * nmemb == 0)
883  return 0;
884 
886 
888 
889  std::string_view hdr( ptr, size*nmemb );
890 
891  hdr.remove_prefix( std::min( hdr.find_first_not_of(" \t\r\n"), hdr.size() ) );
892  const auto lastNonWhitespace = hdr.find_last_not_of(" \t\r\n");
893  if ( lastNonWhitespace != hdr.npos )
894  hdr.remove_suffix( hdr.size() - (lastNonWhitespace + 1) );
895  else
896  hdr = std::string_view();
897 
898  DBG_MEDIA << _easyHandle << " " << "Received header: " << hdr << std::endl;
899 
900  auto &rmode = std::get<running_t>( _runningMode );
901  if ( !hdr.size() ) {
902  return ( size * nmemb );
903  }
904  if ( zypp::strv::hasPrefixCI( hdr, "HTTP/" ) ) {
905 
906  long statuscode = 0;
907  (void)curl_easy_getinfo( _easyHandle, CURLINFO_RESPONSE_CODE, &statuscode);
908 
909  const auto &doRangeFail = [&](){
910  WAR << _easyHandle << " " << "Range FAIL, trying with a smaller batch" << std::endl;
911  rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::RangeFail, "Expected range status code 206, but got none." );
912 
913  // reset all ranges we requested to pending, we never got the data for them
914  std::for_each( _requestedRanges.begin (), _requestedRanges.end(), []( auto &range ) {
915  if ( range._rangeState == NetworkRequest::Running )
916  range._rangeState = NetworkRequest::Pending;
917  });
918  return 0;
919  };
920 
921  // if we have a status 204 we need to create a empty file
922  if( statuscode == 204 && !( _options & NetworkRequest::ConnectionTest ) && !( _options & NetworkRequest::HeadRequest ) )
924 
925  if ( rmode._requireStatusPartial ) {
926  // ignore other status codes, maybe we are redirected etc.
927  if ( ( statuscode >= 200 && statuscode <= 299 && statuscode != 206 )
928  || statuscode == 416 ) {
929  return doRangeFail();
930  }
931  }
932 
933  } else if ( zypp::strv::hasPrefixCI( hdr, "Location:" ) ) {
934  _lastRedirect = hdr.substr( 9 );
935  DBG << _easyHandle << " " << "redirecting to " << _lastRedirect << std::endl;
936 
937  } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Type:") ) {
938  std::string sep;
939  if ( parseContentTypeMultiRangeHeader( hdr, sep ) ) {
940  rmode._gotMultiRangeHeader = true;
941  rmode._seperatorString = "--"+sep;
942  }
943  } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Range:") ) {
945  if ( !parseContentRangeHeader( hdr, r.start, r.len) ) {
946  rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Invalid Content-Range header format." );
947  return 0;
948  }
949  DBG << _easyHandle << " " << "Got content range :" << r.start << " len " << r.len << std::endl;
950  rmode._gotContentRangeHeader = true;
951  rmode._currentSrvRange = r;
952 
953  } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Length:") ) {
954  auto lenStr = str::trim( hdr.substr( 15 ), zypp::str::TRIM );
955  auto str = std::string ( lenStr.data(), lenStr.length() );
956  auto len = zypp::str::strtonum<typename zypp::ByteCount::SizeType>( str.data() );
957  if ( len > 0 ) {
958  DBG << _easyHandle << " " << "Got Content-Length Header: " << len << std::endl;
959  rmode._contentLenght = zypp::ByteCount(len, zypp::ByteCount::B);
960  }
961  }
962  }
963 
964  return ( size * nmemb );
965  }
966 
967  size_t NetworkRequestPrivate::writeCallback(char *ptr, size_t size, size_t nmemb)
968  {
969  const auto max = ( size * nmemb );
970 
972 
973  //it is valid to call this function with no data to write, just return OK
974  if ( max == 0)
975  return 0;
976 
977  //in case of a HEAD request, we do not write anything
979  return ( size * nmemb );
980  }
981 
982  auto &rmode = std::get<running_t>( _runningMode );
983 
984  auto writeDataToFile = [ this, &rmode ]( off_t offset, const char *data, size_t len ) -> off_t {
985 
986  if ( rmode._currentRange < 0 ) {
987  DBG << _easyHandle << " " << "Current range is zero in write request" << std::endl;
988  return 0;
989  }
990 
991  // if we have no open file create or open it
992  if ( !assertOutputFile() )
993  return 0;
994 
995  // seek to the given offset
996  if ( offset >= 0 ) {
997  if ( fseek( rmode._outFile, offset, SEEK_SET ) != 0 ) {
999  "Unable to set output file pointer." );
1000  return 0;
1001  }
1002  }
1003 
1004  auto &rng = _requestedRanges[ rmode._currentRange ];
1005  const auto bytesToWrite = rng.len > 0 ? std::min( rng.len - rng.bytesWritten, len ) : len;
1006 
1007  //make sure we do not write after the expected file size
1008  if ( _expectedFileSize && _expectedFileSize <= static_cast<zypp::ByteCount::SizeType>(rng.start + rng.bytesWritten + bytesToWrite) ) {
1009  rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Downloaded data exceeds expected length." );
1010  return 0;
1011  }
1012 
1013  auto written = fwrite( data, 1, bytesToWrite, rmode._outFile );
1014  if ( written == 0 )
1015  return 0;
1016 
1017  if ( rng._digest && rng._checksum.size() ) {
1018  if ( !rng._digest->update( data, written ) )
1019  return 0;
1020  }
1021 
1022  rng.bytesWritten += written;
1023  if ( rmode._currentSrvRange ) rmode._currentSrvRange->bytesWritten += written;
1024 
1025  if ( rng.len > 0 && rng.bytesWritten >= rng.len ) {
1026  rmode._currentRange = -1;
1027  validateRange( rng );
1028  }
1029 
1030  if ( rmode._currentSrvRange && rmode._currentSrvRange->len > 0 && rmode._currentSrvRange->bytesWritten >= rmode._currentSrvRange->len ) {
1031  rmode._currentSrvRange.reset();
1032  // we ran out of data in the current chunk, reset the target range as well because next data will be
1033  // a chunk header again
1034  rmode._currentRange = -1;
1035  }
1036 
1037  // count the number of real bytes we have downloaded so far
1038  rmode._downloaded += written;
1039  _sigBytesDownloaded.emit( *z_func(), rmode._downloaded );
1040 
1041  return written;
1042  };
1043 
1044  // we are currenty writing a range, continue until we hit the end of the requested chunk, or if we hit end of data
1045  size_t bytesWrittenSoFar = 0;
1046 
1047  while ( bytesWrittenSoFar != max ) {
1048 
1049  off_t seekTo = -1;
1050 
1051  // this is called after all headers have been processed
1052  if ( !rmode._allHeadersReceived ) {
1053  rmode._allHeadersReceived = true;
1054 
1055  // no ranges at all, must be a normal download
1056  if ( !rmode._gotMultiRangeHeader && !rmode._gotContentRangeHeader ) {
1057 
1058  if ( rmode._requireStatusPartial ) {
1059  //we got a invalid response, the status code pointed to being partial but we got no range definition
1061  "Invalid data from server, range respone was announced but there was no range definiton." );
1062  return 0;
1063  }
1064 
1065  //we always download a range even if it is not explicitly requested
1066  if ( _requestedRanges.empty() ) {
1067  _requestedRanges.push_back( NetworkRequest::Range() );
1068  _requestedRanges.back()._rangeState = NetworkRequest::State::Running;
1069  }
1070 
1071  rmode._currentRange = 0;
1072  seekTo = _requestedRanges[0].start;
1073  }
1074  }
1075 
1076  if ( rmode._currentSrvRange && rmode._currentRange == -1 ) {
1077  //if we enter this branch, we just have finished writing a requested chunk but
1078  //are still inside a chunk that was sent by the server, due to the std the server can coalesce requested ranges
1079  //to optimize downloads we need to find the best match ( because the current offset might not even be in our requested ranges )
1080  //Or we just parsed a Content-Lenght header and start a new block
1081 
1082  std::optional<uint> foundRange;
1083  const size_t beginRange = rmode._currentSrvRange->start + rmode._currentSrvRange->bytesWritten;
1084  const size_t endRange = beginRange + (rmode._currentSrvRange->len - rmode._currentSrvRange->bytesWritten);
1085  auto currDist = ULONG_MAX;
1086  for ( uint i = 0; i < _requestedRanges.size(); i++ ) {
1087  const auto &currR = _requestedRanges[i];
1088 
1089  // do not allow double ranges
1090  if ( currR._rangeState == NetworkRequest::Finished || currR._rangeState == NetworkRequest::Error )
1091  continue;
1092 
1093  // check if the range was already written
1094  if ( currR.len == currR.bytesWritten )
1095  continue;
1096 
1097  const auto currRBegin = currR.start + currR.bytesWritten;
1098  if ( !( beginRange <= currRBegin && endRange >= currRBegin ) )
1099  continue;
1100 
1101  // calculate the distance of the current ranges offset+data written to the range we got back from the server
1102  const auto newDist = currRBegin - beginRange;
1103 
1104  if ( !foundRange ) {
1105  foundRange = i;
1106  currDist = newDist;
1107  } else {
1108  //pick the range with the closest distance
1109  if ( newDist < currDist ) {
1110  foundRange = i;
1111  currDist = newDist;
1112  }
1113  }
1114  }
1115  if ( !foundRange ) {
1117  , "Unable to find a matching range for data returned by the server." );
1118  return 0;
1119  }
1120 
1121  //set the found range as the current one
1122  rmode._currentRange = *foundRange;
1123 
1124  //continue writing where we stopped
1125  seekTo = _requestedRanges[*foundRange].start + _requestedRanges[*foundRange].bytesWritten;
1126 
1127  //if we skip bytes we need to advance our written bytecount
1128  const auto skipBytes = seekTo - beginRange;
1129  bytesWrittenSoFar += skipBytes;
1130  rmode._currentSrvRange->bytesWritten += skipBytes;
1131  }
1132 
1133  if ( rmode._currentRange >= 0 ) {
1134  auto availableData = max - bytesWrittenSoFar;
1135  if ( rmode._currentSrvRange ) {
1136  availableData = std::min( availableData, rmode._currentSrvRange->len - rmode._currentSrvRange->bytesWritten );
1137  }
1138  auto bw = writeDataToFile( seekTo, ptr + bytesWrittenSoFar, availableData );
1139  if ( bw <= 0 )
1140  return 0;
1141 
1142  bytesWrittenSoFar += bw;
1143  }
1144 
1145  if ( bytesWrittenSoFar == max )
1146  return max;
1147 
1148  if ( rmode._currentRange == -1 ) {
1149 
1150  // we still are inside the current range from the server
1151  if ( rmode._currentSrvRange )
1152  continue;
1153 
1154  std::string_view incoming( ptr + bytesWrittenSoFar, max - bytesWrittenSoFar );
1155  auto hdrEnd = incoming.find("\r\n\r\n");
1156  if ( hdrEnd == incoming.npos ) {
1157  //no header end in the data yet, push to buffer and return
1158  rmode._rangePrefaceBuffer.insert( rmode._rangePrefaceBuffer.end(), incoming.begin(), incoming.end() );
1159  return max;
1160  }
1161 
1162  //append the data of the current header to the buffer and parse it
1163  rmode._rangePrefaceBuffer.insert( rmode._rangePrefaceBuffer.end(), incoming.begin(), incoming.begin() + ( hdrEnd + 4 ) );
1164  bytesWrittenSoFar += ( hdrEnd + 4 ); //header data plus header end
1165 
1166  std::string_view data( rmode._rangePrefaceBuffer.data(), rmode._rangePrefaceBuffer.size() );
1167  auto sepStrIndex = data.find( rmode._seperatorString );
1168  if ( sepStrIndex == data.npos ) {
1170  "Invalid multirange header format, seperator string missing." );
1171  return 0;
1172  }
1173 
1174  auto startOfHeader = sepStrIndex + rmode._seperatorString.length();
1175  std::vector<std::string_view> lines;
1176  zypp::strv::split( data.substr( startOfHeader ), "\r\n", zypp::strv::Trim::trim, [&]( std::string_view strv ) { lines.push_back(strv); } );
1177  for ( const auto &hdrLine : lines ) {
1178  if ( zypp::strv::hasPrefixCI(hdrLine, "Content-Range:") ) {
1180  //if we can not parse the header the message must be broken
1181  if(! parseContentRangeHeader( hdrLine, r.start, r.len ) ) {
1182  rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Invalid Content-Range header format." );
1183  return 0;
1184  }
1185  rmode._currentSrvRange = r;
1186  break;
1187  }
1188  }
1189  //clear the buffer again
1190  rmode._rangePrefaceBuffer.clear();
1191  }
1192  }
1193  return bytesWrittenSoFar;
1194  }
1195 
1197 
1198  NetworkRequest::NetworkRequest(zyppng::Url url, zypp::filesystem::Pathname targetFile, zyppng::NetworkRequest::FileMode fMode)
1199  : Base ( *new NetworkRequestPrivate( std::move(url), std::move(targetFile), std::move(fMode), *this ) )
1200  {
1201  }
1202 
1204  {
1205  Z_D();
1206 
1207  if ( d->_dispatcher )
1208  d->_dispatcher->cancel( *this, "Request destroyed while still running" );
1209  }
1210 
1212  {
1213  d_func()->_expectedFileSize = std::move( expectedFileSize );
1214  }
1215 
1216  void NetworkRequest::setPriority( NetworkRequest::Priority prio, bool triggerReschedule )
1217  {
1218  Z_D();
1219  d->_priority = prio;
1220  if ( state() == Pending && triggerReschedule && d->_dispatcher )
1221  d->_dispatcher->reschedule();
1222  }
1223 
1225  {
1226  return d_func()->_priority;
1227  }
1228 
1229  void NetworkRequest::setOptions( Options opt )
1230  {
1231  d_func()->_options = opt;
1232  }
1233 
1234  NetworkRequest::Options NetworkRequest::options() const
1235  {
1236  return d_func()->_options;
1237  }
1238 
1239  void NetworkRequest::addRequestRange( size_t start, size_t len, DigestPtr digest, CheckSumBytes expectedChkSum , std::any userData, std::optional<size_t> digestCompareLen, std::optional<size_t> chksumpad )
1240  {
1241  Z_D();
1242  if ( state() == Running )
1243  return;
1244 
1245  d->_requestedRanges.push_back( Range::make( start, len, std::move(digest), std::move( expectedChkSum ), std::move( userData ), digestCompareLen, chksumpad ) );
1246  }
1247 
1249  {
1250  Z_D();
1251  if ( state() == Running )
1252  return;
1253 
1254  d->_requestedRanges.push_back( range );
1255  auto &rng = d->_requestedRanges.back();
1256  rng._rangeState = NetworkRequest::Pending;
1257  rng.bytesWritten = 0;
1258  if ( rng._digest )
1259  rng._digest->reset();
1260  }
1261 
1263  {
1264  Z_D();
1265  if ( state() == Running )
1266  return;
1267  d->_requestedRanges.clear();
1268  }
1269 
1270  std::vector<NetworkRequest::Range> NetworkRequest::failedRanges() const
1271  {
1272  const auto mystate = state();
1273  if ( mystate != Finished && mystate != Error )
1274  return {};
1275 
1276  Z_D();
1277 
1278  std::vector<Range> failed;
1279  for ( const auto &r : d->_requestedRanges ) {
1280  if ( r._rangeState != NetworkRequest::Finished )
1281  failed.push_back( r );
1282  }
1283  return failed;
1284  }
1285 
1286  const std::vector<NetworkRequest::Range> &NetworkRequest::requestedRanges() const
1287  {
1288  return d_func()->_requestedRanges;
1289  }
1290 
1291  const std::string &NetworkRequest::lastRedirectInfo() const
1292  {
1293  return d_func()->_lastRedirect;
1294  }
1295 
1297  {
1298  return d_func()->_easyHandle;
1299  }
1300 
1301  std::optional<zyppng::NetworkRequest::Timings> NetworkRequest::timings() const
1302  {
1303  const auto myerr = error();
1304  const auto mystate = state();
1305  if ( mystate != Finished )
1306  return {};
1307 
1308  Timings t;
1309 
1310  auto getMeasurement = [ this ]( const CURLINFO info, std::chrono::microseconds &target ){
1311  using FPSeconds = std::chrono::duration<double, std::chrono::seconds::period>;
1312  double val = 0;
1313  const auto res = curl_easy_getinfo( d_func()->_easyHandle, info, &val );
1314  if ( CURLE_OK == res ) {
1315  target = std::chrono::duration_cast<std::chrono::microseconds>( FPSeconds(val) );
1316  }
1317  };
1318 
1319  getMeasurement( CURLINFO_NAMELOOKUP_TIME, t.namelookup );
1320  getMeasurement( CURLINFO_CONNECT_TIME, t.connect);
1321  getMeasurement( CURLINFO_APPCONNECT_TIME, t.appconnect);
1322  getMeasurement( CURLINFO_PRETRANSFER_TIME , t.pretransfer);
1323  getMeasurement( CURLINFO_TOTAL_TIME, t.total);
1324  getMeasurement( CURLINFO_REDIRECT_TIME, t.redirect);
1325 
1326  return t;
1327  }
1328 
1329  std::vector<char> NetworkRequest::peekData( off_t offset, size_t count ) const
1330  {
1331  Z_D();
1332 
1333  if ( !std::holds_alternative<NetworkRequestPrivate::running_t>( d->_runningMode) )
1334  return {};
1335 
1336  const auto &rmode = std::get<NetworkRequestPrivate::running_t>( d->_runningMode );
1337  return peek_data_fd( rmode._outFile, offset, count );
1338  }
1339 
1341  {
1342  return d_func()->_url;
1343  }
1344 
1345  void NetworkRequest::setUrl(const Url &url)
1346  {
1347  Z_D();
1348  if ( state() == NetworkRequest::Running )
1349  return;
1350 
1351  d->_url = url;
1352  }
1353 
1355  {
1356  return d_func()->_targetFile;
1357  }
1358 
1360  {
1361  Z_D();
1362  if ( state() == NetworkRequest::Running )
1363  return;
1364  d->_targetFile = path;
1365  }
1366 
1368  {
1369  return d_func()->_fMode;
1370  }
1371 
1373  {
1374  Z_D();
1375  if ( state() == NetworkRequest::Running )
1376  return;
1377  d->_fMode = std::move( mode );
1378  }
1379 
1380  std::string NetworkRequest::contentType() const
1381  {
1382  char *ptr = NULL;
1383  if ( curl_easy_getinfo( d_func()->_easyHandle, CURLINFO_CONTENT_TYPE, &ptr ) == CURLE_OK && ptr )
1384  return std::string(ptr);
1385  return std::string();
1386  }
1387 
1389  {
1390  return std::visit([](auto& arg) -> zypp::ByteCount {
1391  using T = std::decay_t<decltype(arg)>;
1392  if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
1393  return zypp::ByteCount(0);
1394  else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
1395  || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
1396  return arg._contentLenght;
1397  else
1398  static_assert(always_false<T>::value, "Unhandled state type");
1399  }, d_func()->_runningMode);
1400  }
1401 
1403  {
1404  return std::visit([](auto& arg) -> zypp::ByteCount {
1405  using T = std::decay_t<decltype(arg)>;
1406  if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
1407  return zypp::ByteCount();
1408  else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
1409  || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t>
1410  || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
1411  return arg._downloaded;
1412  else
1413  static_assert(always_false<T>::value, "Unhandled state type");
1414  }, d_func()->_runningMode);
1415  }
1416 
1418  {
1419  return d_func()->_settings;
1420  }
1421 
1423  {
1424  return std::visit([this](auto& arg) {
1425  using T = std::decay_t<decltype(arg)>;
1426  if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
1427  return Pending;
1428  else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
1429  return Running;
1430  else if constexpr (std::is_same_v<T, NetworkRequestPrivate::finished_t>) {
1431  if ( std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode )._result.isError() )
1432  return Error;
1433  else
1434  return Finished;
1435  }
1436  else
1437  static_assert(always_false<T>::value, "Unhandled state type");
1438  }, d_func()->_runningMode);
1439  }
1440 
1442  {
1443  const auto s = state();
1444  if ( s != Error && s != Finished )
1445  return NetworkRequestError();
1446  return std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode)._result;
1447  }
1448 
1450  {
1451  if ( !hasError() )
1452  return std::string();
1453 
1454  return error().nativeErrorString();
1455  }
1456 
1458  {
1459  return error().isError();
1460  }
1461 
1462  bool NetworkRequest::addRequestHeader( const std::string &header )
1463  {
1464  Z_D();
1465 
1466  curl_slist *res = curl_slist_append( d->_headers ? d->_headers.get() : nullptr, header.c_str() );
1467  if ( !res )
1468  return false;
1469 
1470  if ( !d->_headers )
1471  d->_headers = std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( res, &curl_slist_free_all );
1472 
1473  return true;
1474  }
1475 
1476  SignalProxy<void (NetworkRequest &req)> NetworkRequest::sigStarted()
1477  {
1478  return d_func()->_sigStarted;
1479  }
1480 
1481  SignalProxy<void (NetworkRequest &req, zypp::ByteCount count)> NetworkRequest::sigBytesDownloaded()
1482  {
1483  return d_func()->_sigBytesDownloaded;
1484  }
1485 
1486  SignalProxy<void (NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> NetworkRequest::sigProgress()
1487  {
1488  return d_func()->_sigProgress;
1489  }
1490 
1491  SignalProxy<void (zyppng::NetworkRequest &req, const zyppng::NetworkRequestError &err)> NetworkRequest::sigFinished()
1492  {
1493  return d_func()->_sigFinished;
1494  }
1495 
1496 }
Signal< void(NetworkRequest &req)> _sigStarted
Definition: request_p.h:133
long timeout() const
transfer timeout
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
std::string errorMessage() const
Definition: request.cc:839
bool isError() const
isError Will return true if this is a actual error
#define MIL
Definition: Logger.h:96
void dump(const char *text, FILE *stream, unsigned char *ptr, size_t size)
Definition: request.cc:36
void setCurlOption(CURLoption opt, T data)
Definition: request_p.h:107
std::optional< Timings > timings() const
After the request is finished query the timings that were collected during download.
Definition: request.cc:1301
void * nativeHandle() const
Definition: request.cc:1296
std::optional< size_t > _chksumPad
Definition: request.h:88
#define DBG_MEDIA
Definition: mediadebug_p.h:28
unsigned size() const
Definition: Regex.cc:106
zypp::ByteCount reportedByteCount() const
Returns the number of bytes that are reported from the backend as the full download size...
Definition: request.cc:1388
const std::vector< Range > & requestedRanges() const
Definition: request.cc:1286
const Pathname & clientCertificatePath() const
SSL client certificate file.
std::chrono::microseconds connect
Definition: request.h:98
std::array< char, CURL_ERROR_SIZE+1 > _errorBuf
Definition: request_p.h:104
void addRequestRange(size_t start, size_t len=0, DigestPtr digest=nullptr, CheckSumBytes expectedChkSum=CheckSumBytes(), std::any userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > chksumpad={})
Definition: request.cc:1239
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar"
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
Regular expression.
Definition: Regex.h:94
ZYPP_IMPL_PRIVATE(Provide)
std::optional< size_t > _relevantDigestLen
Definition: request.h:87
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count)> sigBytesDownloaded()
Signals that new data has been downloaded, this is only the payload and does not include control data...
Definition: request.cc:1481
bool hasPrefixCI(const C_Str &str_r, const C_Str &prefix_r)
Definition: String.h:1030
int log_curl(CURL *, curl_infotype info, char *ptr, size_t len, void *max_lvl)
Definition: curlhelper.cc:68
NetworkRequest::FileMode _fMode
Definition: request_p.h:122
bool checkIfRangeChkSumIsValid(const NetworkRequest::Range &rng)
Definition: request.cc:773
Store and operate with byte count.
Definition: ByteCount.h:30
const std::string & lastRedirectInfo() const
Definition: request.cc:1291
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
const std::string _currentCookieFile
Definition: request_p.h:127
std::chrono::microseconds pretransfer
Definition: request.h:100
Holds transfer setting.
zypp::ByteCount downloadedByteCount() const
Returns the number of already downloaded bytes as reported by the backend.
Definition: request.cc:1402
const std::string & authType() const
get the allowed authentication types
NetworkRequest::Options _options
Definition: request_p.h:118
bool verifyHostEnabled() const
Whether to verify host for ssl.
const std::string & proxyUsername() const
proxy auth username
const char * c_str() const
String representation.
Definition: Pathname.h:110
String related utilities and Regular expression matching.
Definition: Arch.h:351
void setConnectTimeout(long t)
set the connect timeout
std::chrono::microseconds appconnect
Definition: request.h:99
bool prepareNextRangeBatch(std::string &errBuf)
Definition: request.cc:562
constexpr bool always_false
Definition: PathInfo.cc:544
running_t(pending_t &&prevState)
Definition: request.cc:174
std::string nativeErrorString() const
Signal< void(NetworkRequest &req, zypp::ByteCount count)> _sigBytesDownloaded
Definition: request_p.h:134
Convenient building of std::string with boost::format.
Definition: String.h:252
Structure holding values of curlrc options.
Definition: curlconfig.h:26
void setOptions(Options opt)
Definition: request.cc:1229
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:36
TransferSettings & transferSettings()
Definition: request.cc:1417
enum zyppng::NetworkRequestPrivate::ProtocolMode _protocolMode
void setExpectedFileSize(zypp::ByteCount expectedFileSize)
Definition: request.cc:1211
void setFileOpenMode(FileMode mode)
Sets the file open mode to mode.
Definition: request.cc:1372
bool hasError() const
Checks if there was a error with the request.
Definition: request.cc:1457
void onActivityTimeout(Timer &)
Definition: request.cc:762
const Headers & headers() const
returns a list of all added headers
static std::string digestVectorToString(const UByteArray &vec)
get hex string representation of the digest vector given as parameter
Definition: Digest.cc:184
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:48
zypp::Pathname _targetFile
Definition: request_p.h:116
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
void setUrl(const Url &url)
This will change the URL of the request.
Definition: request.cc:1345
std::chrono::microseconds namelookup
Definition: request.h:97
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: curlconfig.cc:24
Do not differentiate case.
Definition: Regex.h:99
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \, const Trim trim_r=NO_TRIM)
Split line_r into words.
Definition: String.h:531
size_t headerCallback(char *ptr, size_t size, size_t nmemb)
Definition: request.cc:879
Convenient building of std::string via std::ostringstream Basically a std::ostringstream autoconverti...
Definition: String.h:211
bool addRequestHeader(const std::string &header)
Definition: request.cc:1462
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:223
const std::string & asString() const
String representation.
Definition: Pathname.h:91
Signal< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> _sigProgress
Definition: request_p.h:135
bool parseContentTypeMultiRangeHeader(const std::string_view &line, std::string &boundary)
Definition: request.cc:825
std::string asString() const
Error message provided by dumpOn as string.
Definition: Exception.cc:75
long connectTimeout() const
connection timeout
bool initialize(std::string &errBuf)
Definition: request.cc:203
#define WAR
Definition: Logger.h:97
#define nullptr
Definition: Easy.h:55
The NetworkRequestError class Represents a error that occured in.
std::vector< char > peekData(off_t offset, size_t count) const
Definition: request.cc:1329
NetworkRequestError error() const
Returns the last set Error.
Definition: request.cc:1441
int log_curl_all(CURL *handle, curl_infotype type, char *data, size_t size, void *userp)
Definition: request.cc:67
zypp::ByteCount _expectedFileSize
Definition: request_p.h:119
static constexpr int _rangeAttempt[]
Definition: request_p.h:149
UByteArray CheckSumBytes
Definition: request.h:47
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
Definition: request.cc:1449
void setTimeout(long t)
set the transfer timeout
Priority priority() const
Definition: request.cc:1224
std::string proxyuserpwd
Definition: curlconfig.h:49
bool setupHandle(std::string &errBuf)
Definition: request.cc:215
const Pathname & clientKeyPath() const
SSL client key file.
const zypp::Pathname & targetFilePath() const
Returns the target filename path.
Definition: request.cc:1354
void validateRange(NetworkRequest::Range &rng)
Definition: request.cc:793
std::unique_ptr< curl_slist, decltype(&curl_slist_free_all) > _headers
Definition: request_p.h:142
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
#define MIL_MEDIA
Definition: mediadebug_p.h:29
bool parseContentRangeHeader(const std::string_view &line, size_t &start, size_t &len)
Definition: request.cc:808
std::vector< char > peek_data_fd(FILE *fd, off_t offset, size_t count)
Definition: request.cc:135
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:436
bool proxyEnabled() const
proxy is enabled
void setTargetFilePath(const zypp::Pathname &path)
Changes the target file path of the download.
Definition: request.cc:1359
static int curlProgressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Definition: request.cc:853
Regular expression match result.
Definition: Regex.h:167
std::string contentType() const
Returns the content type as reported from the server.
Definition: request.cc:1380
static const Unit B
1 Byte
Definition: ByteCount.h:42
Base class for Exception.
Definition: Exception.h:145
std::string _lastRedirect
to log/report redirections
Definition: request_p.h:126
std::chrono::microseconds total
Definition: request.h:101
bool any_of(const Container &c, Fnc &&cb)
Definition: Algorithm.h:76
CheckSumBytes _checksum
Enables automated checking of downloaded contents against a checksum.
Definition: request.h:86
std::string curlUnEscape(std::string text_r)
Definition: curlhelper.cc:325
static Range make(size_t start, size_t len=0, DigestPtr &&digest=nullptr, CheckSumBytes &&expectedChkSum=CheckSumBytes(), std::any &&userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > _dataBlockPadding={})
Definition: request.cc:153
void setPriority(Priority prio, bool triggerReschedule=true)
Definition: request.cc:1216
State state() const
Returns the current state the HttpDownloadRequest is in.
Definition: request.cc:1422
TransferSettings _settings
Definition: request_p.h:117
NetworkRequestDispatcher * _dispatcher
Definition: request_p.h:130
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
virtual ~NetworkRequest()
Definition: request.cc:1203
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3"
Options options() const
Definition: request.cc:1234
std::vector< NetworkRequest::Range > _requestedRanges
the requested ranges that need to be downloaded
Definition: request_p.h:120
size_t writeCallback(char *ptr, size_t size, size_t nmemb)
Definition: request.cc:967
bool regex_match(const std::string &s, smatch &matches, const regex &regex)
regex ZYPP_STR_REGEX regex ZYPP_STR_REGEX
Definition: Regex.h:70
std::chrono::microseconds redirect
Definition: request.h:102
std::shared_ptr< zypp::Digest > DigestPtr
Definition: request.h:46
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
Definition: request.cc:1491
Signal< void(NetworkRequest &req, const NetworkRequestError &err)> _sigFinished
Definition: request_p.h:136
Type type() const
type Returns the type of the error
These are enforced even if you don&#39;t pass them as flag argument.
Definition: Regex.h:103
SignalProxy< void(NetworkRequest &req)> sigStarted()
Signals that the dispatcher dequeued the request and actually starts downloading data.
Definition: request.cc:1476
std::string userPassword() const
returns the user and password as a user:pass string
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:25
FileMode fileOpenMode() const
Returns the currently configured file open mode.
Definition: request.cc:1367
std::vector< Range > failedRanges() const
Definition: request.cc:1270
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
NetworkRequestPrivate(Url &&url, zypp::Pathname &&targetFile, NetworkRequest::FileMode fMode, NetworkRequest &p)
Definition: request.cc:185
void setResult(NetworkRequestError &&err)
Definition: request.cc:716
#define CONNECT_TIMEOUT
Definition: curlhelper_p.h:21
const std::string & proxy() const
proxy host
static zyppng::NetworkRequestError customError(NetworkRequestError::Type t, std::string &&errorMsg="", std::map< std::string, boost::any > &&extraInfo={})
SignalProxy< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> sigProgress()
Signals if there was data read from the download.
Definition: request.cc:1486
bool prepareToContinue(std::string &errBuf)
Definition: request.cc:530
const std::string & userAgentString() const
user agent string
bool headRequestsAllowed() const
whether HEAD requests are allowed
static MediaConfig & instance()
Definition: mediaconfig.cc:43
#define DBG
Definition: Logger.h:95
ZYppCommitResult & _result
Definition: TargetImpl.cc:1564
std::variant< pending_t, running_t, prepareNextRangeBatch_t, finished_t > _runningMode
Definition: request_p.h:212