libzypp  17.31.22
MediaCurl.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
13 #include <iostream>
14 #include <chrono>
15 #include <list>
16 
17 #include <zypp/base/Logger.h>
18 #include <zypp/ExternalProgram.h>
19 #include <zypp/base/String.h>
20 #include <zypp/base/Gettext.h>
21 #include <zypp-core/parser/Sysconfig>
22 #include <zypp/base/Gettext.h>
23 
24 #include <zypp/media/MediaCurl.h>
25 #include <zypp-curl/ProxyInfo>
26 #include <zypp-curl/auth/CurlAuthData>
27 #include <zypp-media/auth/CredentialManager>
28 #include <zypp-curl/CurlConfig>
30 #include <zypp/Target.h>
31 #include <zypp/ZYppFactory.h>
32 #include <zypp/ZConfig.h>
33 
34 #include <cstdlib>
35 #include <sys/types.h>
36 #include <sys/stat.h>
37 #include <sys/mount.h>
38 #include <errno.h>
39 #include <dirent.h>
40 #include <unistd.h>
41 
42 using std::endl;
43 
44 namespace internal {
45  using namespace zypp;
49  struct OptionalDownloadProgressReport : public callback::ReceiveReport<media::DownloadProgressReport>
50  {
51  using TimePoint = std::chrono::steady_clock::time_point;
52 
53  OptionalDownloadProgressReport( bool isOptional=false )
54  : _oldRec { Distributor::instance().getReceiver() }
55  , _isOptional { isOptional }
56  { connect(); }
57 
59  { if ( _oldRec ) Distributor::instance().setReceiver( *_oldRec ); else Distributor::instance().noReceiver(); }
60 
61  void reportbegin() override
62  { if ( _oldRec ) _oldRec->reportbegin(); }
63 
64  void reportend() override
65  { if ( _oldRec ) _oldRec->reportend(); }
66 
67  void report( const UserData & userData_r = UserData() ) override
68  { if ( _oldRec ) _oldRec->report( userData_r ); }
69 
70 
71  void start( const Url & file_r, Pathname localfile_r ) override
72  {
73  if ( not _oldRec ) return;
74  if ( _isOptional ) {
75  // delay start until first data are received.
76  _startFile = file_r;
77  _startLocalfile = std::move(localfile_r);
78  return;
79  }
80  _oldRec->start( file_r, localfile_r );
81  }
82 
83  bool progress( int value_r, const Url & file_r, double dbps_avg_r = -1, double dbps_current_r = -1 ) override
84  {
85  if ( not _oldRec ) return true;
86  if ( notStarted() ) {
87  if ( not ( value_r || dbps_avg_r || dbps_current_r ) )
88  return true;
89  sendStart();
90  }
91 
92  //static constexpr std::chrono::milliseconds minfequency { 1000 }; only needed if we'd avoid sending reports without change
93  static constexpr std::chrono::milliseconds maxfequency { 100 };
94  TimePoint now { TimePoint::clock::now() };
95  TimePoint::duration elapsed { now - _lastProgressSent };
96  if ( elapsed < maxfequency )
97  return true; // continue
98  _lastProgressSent = now;
99  return _oldRec->progress( value_r, file_r, dbps_avg_r, dbps_current_r );
100  }
101 
102  Action problem( const Url & file_r, Error error_r, const std::string & description_r ) override
103  {
104  if ( not _oldRec || notStarted() ) return ABORT;
105  return _oldRec->problem( file_r, error_r, description_r );
106  }
107 
108  void finish( const Url & file_r, Error error_r, const std::string & reason_r ) override
109  {
110  if ( not _oldRec || notStarted() ) return;
111  _oldRec->finish( file_r, error_r, reason_r );
112  }
113 
114  private:
115  // _isOptional also indicates the delayed start
116  bool notStarted() const
117  { return _isOptional; }
118 
119  void sendStart()
120  {
121  if ( _isOptional ) {
122  // we know _oldRec is valid...
123  _oldRec->start( std::move(_startFile), std::move(_startLocalfile) );
124  _isOptional = false;
125  }
126  }
127 
128  private:
129  Receiver *const _oldRec;
132  Pathname _startLocalfile;
134  };
135 
137  {
138  ProgressData( CURL *curl, time_t timeout = 0, const zypp::Url & url = zypp::Url(),
139  zypp::ByteCount expectedFileSize_r = 0,
141 
142  void updateStats( double dltotal = 0.0, double dlnow = 0.0 );
143 
144  int reportProgress() const;
145 
146  CURL * curl()
147  { return _curl; }
148 
149  bool timeoutReached() const
150  { return _timeoutReached; }
151 
152  bool fileSizeExceeded() const
153  { return _fileSizeExceeded; }
154 
156  { return _expectedFileSize; }
157 
158  void expectedFileSize( ByteCount newval_r )
159  { _expectedFileSize = newval_r; }
160 
161  private:
162  CURL * _curl;
164  time_t _timeout;
169 
170  time_t _timeStart = 0;
171  time_t _timeLast = 0;
172  time_t _timeRcv = 0;
173  time_t _timeNow = 0;
174 
175  double _dnlTotal = 0.0;
176  double _dnlLast = 0.0;
177  double _dnlNow = 0.0;
178 
179  int _dnlPercent= 0;
180 
181  double _drateTotal= 0.0;
182  double _drateLast = 0.0;
183  };
184 
185 
186 
187  ProgressData::ProgressData(CURL *curl, time_t timeout, const Url &url, ByteCount expectedFileSize_r, zypp::callback::SendReport< zypp::media::DownloadProgressReport> *_report)
188  : _curl( curl )
189  , _url( url )
190  , _timeout( timeout )
191  , _timeoutReached( false )
192  , _fileSizeExceeded ( false )
193  , _expectedFileSize( expectedFileSize_r )
194  , report( _report )
195  {}
196 
197  void ProgressData::updateStats(double dltotal, double dlnow)
198  {
199  time_t now = _timeNow = time(0);
200 
201  // If called without args (0.0), recompute based on the last values seen
202  if ( dltotal && dltotal != _dnlTotal )
203  _dnlTotal = dltotal;
204 
205  if ( dlnow && dlnow != _dnlNow )
206  {
207  _timeRcv = now;
208  _dnlNow = dlnow;
209  }
210 
211  // init or reset if time jumps back
212  if ( !_timeStart || _timeStart > now )
213  _timeStart = _timeLast = _timeRcv = now;
214 
215  // timeout condition
216  if ( _timeout )
217  _timeoutReached = ( (now - _timeRcv) > _timeout );
218 
219  // check if the downloaded data is already bigger than what we expected
220  _fileSizeExceeded = _expectedFileSize > 0 && _expectedFileSize < static_cast<ByteCount::SizeType>(_dnlNow);
221 
222  // percentage:
223  if ( _dnlTotal )
224  _dnlPercent = int(_dnlNow * 100 / _dnlTotal);
225 
226  // download rates:
227  _drateTotal = _dnlNow / std::max( int(now - _timeStart), 1 );
228 
229  if ( _timeLast < now )
230  {
231  _drateLast = (_dnlNow - _dnlLast) / int(now - _timeLast);
232  // start new period
233  _timeLast = now;
234  _dnlLast = _dnlNow;
235  }
236  else if ( _timeStart == _timeLast )
238  }
239 
241  {
242  if ( _fileSizeExceeded )
243  return 1;
244  if ( _timeoutReached )
245  return 1; // no-data timeout
246  if ( report && !(*report)->progress( _dnlPercent, _url, _drateTotal, _drateLast ) )
247  return 1; // user requested abort
248  return 0;
249  }
250 
251  const char * anonymousIdHeader()
252  {
253  // we need to add the release and identifier to the
254  // agent string.
255  // The target could be not initialized, and then this information
256  // is guessed.
257  // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
258  static const std::string _value( str::trim( str::form(
259  "X-ZYpp-AnonymousId: %s",
260  Target::anonymousUniqueId( Pathname()/*guess root*/ ).c_str()
261  )));
262  return _value.c_str();
263  }
264 
266  {
267  // we need to add the release and identifier to the
268  // agent string.
269  // The target could be not initialized, and then this information
270  // is guessed.
271  // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
272  static const std::string _value( str::trim( str::form(
273  "X-ZYpp-DistributionFlavor: %s",
274  Target::distributionFlavor( Pathname()/*guess root*/ ).c_str()
275  )));
276  return _value.c_str();
277  }
278 
279  const char * agentString()
280  {
281  // we need to add the release and identifier to the
282  // agent string.
283  // The target could be not initialized, and then this information
284  // is guessed.
285  // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
286  static const std::string _value( str::trim( str::form(
287  "ZYpp " LIBZYPP_VERSION_STRING " (curl %s) %s"
288  , curl_version_info(CURLVERSION_NOW)->version
289  , Target::targetDistribution( Pathname()/*guess root*/ ).c_str()
290  )));
291  return _value.c_str();
292  }
293 
298  {
299  public:
301  const std::string & err_r,
302  const std::string & msg_r )
303  : media::MediaCurlException( url_r, err_r, msg_r )
304  {}
305  //~MediaCurlExceptionMayRetryInternaly() noexcept {}
306  };
307 
308 }
309 
310 
311 using namespace internal;
312 using namespace zypp::base;
313 
314 namespace zypp {
315 
316  namespace media {
317 
318 Pathname MediaCurl::_cookieFile = "/var/lib/YaST2/cookies";
319 
320 // we use this define to unbloat code as this C setting option
321 // and catching exception is done frequently.
323 #define SET_OPTION(opt,val) do { \
324  ret = curl_easy_setopt ( _curl, opt, val ); \
325  if ( ret != 0) { \
326  ZYPP_THROW(MediaCurlSetOptException(_url, _curlError)); \
327  } \
328  } while ( false )
329 
330 #define SET_OPTION_OFFT(opt,val) SET_OPTION(opt,(curl_off_t)val)
331 #define SET_OPTION_LONG(opt,val) SET_OPTION(opt,(long)val)
332 #define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val)
333 
334 MediaCurl::MediaCurl( const Url & url_r,
335  const Pathname & attach_point_hint_r )
336  : MediaNetworkCommonHandler( url_r, attach_point_hint_r,
337  "/", // urlpath at attachpoint
338  true ), // does_download
339  _curl( NULL ),
340  _customHeaders(0L)
341 {
342  _curlError[0] = '\0';
343 
344  MIL << "MediaCurl::MediaCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl;
345 
347 
348  if( !attachPoint().empty())
349  {
350  PathInfo ainfo(attachPoint());
351  Pathname apath(attachPoint() + "XXXXXX");
352  char *atemp = ::strdup( apath.asString().c_str());
353  char *atest = NULL;
354  if( !ainfo.isDir() || !ainfo.userMayRWX() ||
355  atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
356  {
357  WAR << "attach point " << ainfo.path()
358  << " is not useable for " << url_r.getScheme() << endl;
359  setAttachPoint("", true);
360  }
361  else if( atest != NULL)
362  ::rmdir(atest);
363 
364  if( atemp != NULL)
365  ::free(atemp);
366  }
367 }
368 
370 {
372 }
373 
374 void MediaCurl::setCookieFile( const Pathname &fileName )
375 {
376  _cookieFile = fileName;
377 }
378 
380 
381 void MediaCurl::checkProtocol(const Url &url) const
382 {
383  curl_version_info_data *curl_info = NULL;
384  curl_info = curl_version_info(CURLVERSION_NOW);
385  // curl_info does not need any free (is static)
386  if (curl_info->protocols)
387  {
388  const char * const *proto;
389  std::string scheme( url.getScheme());
390  bool found = false;
391  for(proto=curl_info->protocols; !found && *proto; ++proto)
392  {
393  if( scheme == std::string((const char *)*proto))
394  found = true;
395  }
396  if( !found)
397  {
398  std::string msg("Unsupported protocol '");
399  msg += scheme;
400  msg += "'";
402  }
403  }
404 }
405 
407 {
409 
410  curl_easy_setopt(_curl, CURLOPT_HEADERFUNCTION, log_redirects_curl);
411  curl_easy_setopt(_curl, CURLOPT_HEADERDATA, &_lastRedirect);
412  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_ERRORBUFFER, _curlError );
413  if ( ret != 0 ) {
414  ZYPP_THROW(MediaCurlSetOptException(_url, "Error setting error buffer"));
415  }
416 
417  SET_OPTION(CURLOPT_FAILONERROR, 1L);
418  SET_OPTION(CURLOPT_NOSIGNAL, 1L);
419 
420  // create non persistant settings
421  // so that we don't add headers twice
422  TransferSettings vol_settings(_settings);
423 
424  // add custom headers for download.opensuse.org (bsc#955801)
425  if ( _url.getHost() == "download.opensuse.org" )
426  {
427  vol_settings.addHeader(anonymousIdHeader());
428  vol_settings.addHeader(distributionFlavorHeader());
429  }
430  vol_settings.addHeader("Pragma:");
431 
433 
434  // fill some settings from url query parameters
435  try
436  {
438  }
439  catch ( const MediaException &e )
440  {
441  disconnectFrom();
442  ZYPP_RETHROW(e);
443  }
444  // if the proxy was not set (or explicitly unset) by url, then look...
445  if ( _settings.proxy().empty() )
446  {
447  // ...at the system proxy settings
449  }
450 
452  switch ( env::ZYPP_MEDIA_CURL_IPRESOLVE() )
453  {
454  case 4: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); break;
455  case 6: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); break;
456  }
457 
461  SET_OPTION(CURLOPT_CONNECTTIMEOUT, _settings.connectTimeout());
462  // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
463  // just in case curl does not trigger its progress callback frequently
464  // enough.
465  if ( _settings.timeout() )
466  {
467  SET_OPTION(CURLOPT_TIMEOUT, 3600L);
468  }
469 
470  // follow any Location: header that the server sends as part of
471  // an HTTP header (#113275)
472  SET_OPTION(CURLOPT_FOLLOWLOCATION, 1L);
473  // 3 redirects seem to be too few in some cases (bnc #465532)
474  SET_OPTION(CURLOPT_MAXREDIRS, 6L);
475 
476  if ( _url.getScheme() == "https" )
477  {
478 #if CURLVERSION_AT_LEAST(7,19,4)
479  // restrict following of redirections from https to https only
480  if ( _url.getHost() == "download.opensuse.org" )
481  SET_OPTION( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS );
482  else
483  SET_OPTION( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
484 #endif
485 
488  {
490  }
491 
493  {
494  SET_OPTION(CURLOPT_SSLCERT, _settings.clientCertificatePath().c_str());
495  }
496  if( ! _settings.clientKeyPath().empty() )
497  {
498  SET_OPTION(CURLOPT_SSLKEY, _settings.clientKeyPath().c_str());
499  }
500 
501 #ifdef CURLSSLOPT_ALLOW_BEAST
502  // see bnc#779177
503  ret = curl_easy_setopt( _curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
504  if ( ret != 0 ) {
505  disconnectFrom();
507  }
508 #endif
509  SET_OPTION(CURLOPT_SSL_VERIFYPEER, _settings.verifyPeerEnabled() ? 1L : 0L);
510  SET_OPTION(CURLOPT_SSL_VERIFYHOST, _settings.verifyHostEnabled() ? 2L : 0L);
511  // bnc#903405 - POODLE: libzypp should only talk TLS
512  SET_OPTION(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
513  }
514 
515  SET_OPTION(CURLOPT_USERAGENT, _settings.userAgentString().c_str() );
516 
517  /* Fixes bsc#1174011 "auth=basic ignored in some cases"
518  * We should proactively add the password to the request if basic auth is configured
519  * and a password is available in the credentials but not in the URL.
520  *
521  * We will be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
522  * and ask the server first about the auth method
523  */
524  if ( _settings.authType() == "basic"
525  && _settings.username().size()
526  && !_settings.password().size() ) {
527 
528  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
529  const auto cred = cm.getCred( _url );
530  if ( cred && cred->valid() ) {
531  if ( !_settings.username().size() )
532  _settings.setUsername(cred->username());
533  _settings.setPassword(cred->password());
534  }
535  }
536 
537  /*---------------------------------------------------------------*
538  CURLOPT_USERPWD: [user name]:[password]
539 
540  Url::username/password -> CURLOPT_USERPWD
541  If not provided, anonymous FTP identification
542  *---------------------------------------------------------------*/
543 
544  if ( _settings.userPassword().size() )
545  {
546  SET_OPTION(CURLOPT_USERPWD, _settings.userPassword().c_str());
547  std::string use_auth = _settings.authType();
548  if (use_auth.empty())
549  use_auth = "digest,basic"; // our default
550  long auth = CurlAuthData::auth_type_str2long(use_auth);
551  if( auth != CURLAUTH_NONE)
552  {
553  DBG << "Enabling HTTP authentication methods: " << use_auth
554  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
555  SET_OPTION(CURLOPT_HTTPAUTH, auth);
556  }
557  }
558 
559  if ( _settings.proxyEnabled() && ! _settings.proxy().empty() )
560  {
561  DBG << "Proxy: '" << _settings.proxy() << "'" << endl;
562  SET_OPTION(CURLOPT_PROXY, _settings.proxy().c_str());
563  SET_OPTION(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
564  /*---------------------------------------------------------------*
565  * CURLOPT_PROXYUSERPWD: [user name]:[password]
566  *
567  * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
568  * If not provided, $HOME/.curlrc is evaluated
569  *---------------------------------------------------------------*/
570 
571  std::string proxyuserpwd = _settings.proxyUserPassword();
572 
573  if ( proxyuserpwd.empty() )
574  {
575  CurlConfig curlconf;
576  CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
577  if ( curlconf.proxyuserpwd.empty() )
578  DBG << "Proxy: ~/.curlrc does not contain the proxy-user option" << endl;
579  else
580  {
581  proxyuserpwd = curlconf.proxyuserpwd;
582  DBG << "Proxy: using proxy-user from ~/.curlrc" << endl;
583  }
584  }
585  else
586  {
587  DBG << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << endl;
588  }
589 
590  if ( ! proxyuserpwd.empty() )
591  {
592  SET_OPTION(CURLOPT_PROXYUSERPWD, curlUnEscape( proxyuserpwd ).c_str());
593  }
594  }
595 #if CURLVERSION_AT_LEAST(7,19,4)
596  else if ( _settings.proxy() == EXPLICITLY_NO_PROXY )
597  {
598  // Explicitly disabled in URL (see fillSettingsFromUrl()).
599  // This should also prevent libcurl from looking into the environment.
600  DBG << "Proxy: explicitly NOPROXY" << endl;
601  SET_OPTION(CURLOPT_NOPROXY, "*");
602  }
603 #endif
604  else
605  {
606  DBG << "Proxy: not explicitly set" << endl;
607  DBG << "Proxy: libcurl may look into the environment" << endl;
608  }
609 
611  if ( _settings.minDownloadSpeed() != 0 )
612  {
613  SET_OPTION(CURLOPT_LOW_SPEED_LIMIT, _settings.minDownloadSpeed());
614  // default to 10 seconds at low speed
615  SET_OPTION(CURLOPT_LOW_SPEED_TIME, 60L);
616  }
617 
618 #if CURLVERSION_AT_LEAST(7,15,5)
619  if ( _settings.maxDownloadSpeed() != 0 )
620  SET_OPTION_OFFT(CURLOPT_MAX_RECV_SPEED_LARGE, _settings.maxDownloadSpeed());
621 #endif
622 
623  /*---------------------------------------------------------------*
624  *---------------------------------------------------------------*/
625 
628  if ( str::strToBool( _url.getQueryParam( "cookies" ), true ) )
629  SET_OPTION(CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
630  else
631  MIL << "No cookies requested" << endl;
632  SET_OPTION(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
633  SET_OPTION(CURLOPT_PROGRESSFUNCTION, &progressCallback );
634  SET_OPTION(CURLOPT_NOPROGRESS, 0L);
635 
636 #if CURLVERSION_AT_LEAST(7,18,0)
637  // bnc #306272
638  SET_OPTION(CURLOPT_PROXY_TRANSFER_MODE, 1L );
639 #endif
640  // Append settings custom headers to curl.
641  // TransferSettings assert strings are trimmed (HTTP/2 RFC 9113)
642  for ( const auto &header : vol_settings.headers() ) {
643  _customHeaders = curl_slist_append(_customHeaders, header.c_str());
644  if ( !_customHeaders )
646  }
647  SET_OPTION(CURLOPT_HTTPHEADER, _customHeaders);
648 }
649 
651 
652 
653 void MediaCurl::attachTo (bool next)
654 {
655  if ( next )
657 
658  if ( !_url.isValid() )
660 
663  {
665  }
666 
667  disconnectFrom(); // clean _curl if needed
668  _curl = curl_easy_init();
669  if ( !_curl ) {
671  }
672  try
673  {
674  setupEasy();
675  }
676  catch (Exception & ex)
677  {
678  disconnectFrom();
679  ZYPP_RETHROW(ex);
680  }
681 
682  // FIXME: need a derived class to propelly compare url's
684  setMediaSource(media);
685 }
686 
687 bool
689 {
690  return MediaHandler::checkAttachPoint( apoint, true, true);
691 }
692 
694 
696 {
697  if ( _customHeaders )
698  {
699  curl_slist_free_all(_customHeaders);
700  _customHeaders = 0L;
701  }
702 
703  if ( _curl )
704  {
705  // bsc#1201092: Strange but within global_dtors we may exceptions here.
706  try { curl_easy_cleanup( _curl ); }
707  catch (...) { ; }
708  _curl = NULL;
709  }
710 }
711 
713 
714 void MediaCurl::releaseFrom( const std::string & ejectDev )
715 {
716  disconnect();
717 }
718 
720 
721 void MediaCurl::getFile( const OnMediaLocation &file ) const
722 {
723  // Use absolute file name to prevent access of files outside of the
724  // hierarchy below the attach point.
725  getFileCopy( file, localPath(file.filename()).absolutename() );
726 }
727 
729 
730 void MediaCurl::getFileCopy( const OnMediaLocation & srcFile , const Pathname & target ) const
731 {
732 
733  const auto &filename = srcFile.filename();
734 
735  // Optional files will send no report until data are actually received (we know it exists).
736  OptionalDownloadProgressReport reportfilter( srcFile.optional() );
738 
739  Url fileurl(getFileUrl(filename));
740 
741  bool firstAuth = true; // bsc#1210870: authenticate must not return stored credentials more than once.
742  unsigned internalTry = 0;
743  static constexpr unsigned maxInternalTry = 3;
744 
745  do
746  {
747  try
748  {
749  doGetFileCopy( srcFile, target, report );
750  break; // success!
751  }
752  // retry with proper authentication data
753  catch (MediaUnauthorizedException & ex_r)
754  {
755  if ( authenticate(ex_r.hint(), firstAuth) ) {
756  firstAuth = false; // must not return stored credentials again
757  continue; // retry
758  }
759 
760  report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
761  ZYPP_RETHROW(ex_r);
762  }
763  // unexpected exception
764  catch (MediaException & excpt_r)
765  {
766  if ( typeid(excpt_r) == typeid( MediaCurlExceptionMayRetryInternaly ) ) {
767  ++internalTry;
768  if ( internalTry < maxInternalTry ) {
769  // just report (NO_ERROR); no interactive request to the user
770  report->problem(fileurl, media::DownloadProgressReport::NO_ERROR, excpt_r.asUserHistory()+_("Will try again..."));
771  continue; // retry
772  }
773  excpt_r.addHistory( str::Format(_("Giving up after %1% attempts.")) % maxInternalTry );
774  }
775 
777  if( typeid(excpt_r) == typeid( media::MediaFileNotFoundException ) ||
778  typeid(excpt_r) == typeid( media::MediaNotAFileException ) )
779  {
781  }
782  report->finish(fileurl, reason, excpt_r.asUserHistory());
783  ZYPP_RETHROW(excpt_r);
784  }
785  }
786  while ( true );
787  report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
788 }
789 
791 
792 bool MediaCurl::getDoesFileExist( const Pathname & filename ) const
793 {
794  bool retry = false;
795 
796  do
797  {
798  try
799  {
800  return doGetDoesFileExist( filename );
801  }
802  // authentication problem, retry with proper authentication data
803  catch (MediaUnauthorizedException & ex_r)
804  {
805  if(authenticate(ex_r.hint(), !retry))
806  retry = true;
807  else
808  ZYPP_RETHROW(ex_r);
809  }
810  // unexpected exception
811  catch (MediaException & excpt_r)
812  {
813  ZYPP_RETHROW(excpt_r);
814  }
815  }
816  while (retry);
817 
818  return false;
819 }
820 
822 
824  CURLcode code,
825  bool timeout_reached) const
826 {
827  if ( code != 0 )
828  {
829  Url url;
830  if (filename.empty())
831  url = _url;
832  else
833  url = getFileUrl(filename);
834 
835  std::string err;
836  {
837  switch ( code )
838  {
839  case CURLE_UNSUPPORTED_PROTOCOL:
840  err = " Unsupported protocol";
841  if ( !_lastRedirect.empty() )
842  {
843  err += " or redirect (";
844  err += _lastRedirect;
845  err += ")";
846  }
847  break;
848  case CURLE_URL_MALFORMAT:
849  case CURLE_URL_MALFORMAT_USER:
850  err = " Bad URL";
851  break;
852  case CURLE_LOGIN_DENIED:
853  ZYPP_THROW(
854  MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
855  break;
856  case CURLE_HTTP_RETURNED_ERROR:
857  {
858  long httpReturnCode = 0;
859  CURLcode infoRet = curl_easy_getinfo( _curl,
860  CURLINFO_RESPONSE_CODE,
861  &httpReturnCode );
862  if ( infoRet == CURLE_OK )
863  {
864  std::string msg = "HTTP response: " + str::numstring( httpReturnCode );
865  switch ( httpReturnCode )
866  {
867  case 401:
868  {
869  std::string auth_hint = getAuthHint();
870 
871  DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
872  DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
873 
875  url, "Login failed.", _curlError, auth_hint
876  ));
877  }
878 
879  case 502: // bad gateway (bnc #1070851)
880  case 503: // service temporarily unavailable (bnc #462545)
882  case 504: // gateway timeout
884  case 403:
885  {
886  std::string msg403;
887  if ( url.getHost().find(".suse.com") != std::string::npos )
888  msg403 = _("Visit the SUSE Customer Center to check whether your registration is valid and has not expired.");
889  else if (url.asString().find("novell.com") != std::string::npos)
890  msg403 = _("Visit the Novell Customer Center to check whether your registration is valid and has not expired.");
892  }
893  case 404:
894  case 410:
896  }
897 
898  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
900  }
901  else
902  {
903  std::string msg = "Unable to retrieve HTTP response:";
904  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
906  }
907  }
908  break;
909  case CURLE_FTP_COULDNT_RETR_FILE:
910 #if CURLVERSION_AT_LEAST(7,16,0)
911  case CURLE_REMOTE_FILE_NOT_FOUND:
912 #endif
913  case CURLE_FTP_ACCESS_DENIED:
914  case CURLE_TFTP_NOTFOUND:
915  err = "File not found";
917  break;
918  case CURLE_BAD_PASSWORD_ENTERED:
919  case CURLE_FTP_USER_PASSWORD_INCORRECT:
920  err = "Login failed";
921  break;
922  case CURLE_COULDNT_RESOLVE_PROXY:
923  case CURLE_COULDNT_RESOLVE_HOST:
924  case CURLE_COULDNT_CONNECT:
925  case CURLE_FTP_CANT_GET_HOST:
926  err = "Connection failed";
927  break;
928  case CURLE_WRITE_ERROR:
929  err = "Write error";
930  break;
931  case CURLE_PARTIAL_FILE:
932  case CURLE_OPERATION_TIMEDOUT:
933  timeout_reached = true; // fall though to TimeoutException
934  // fall though...
935  case CURLE_ABORTED_BY_CALLBACK:
936  if( timeout_reached )
937  {
938  err = "Timeout reached";
940  }
941  else
942  {
943  err = "User abort";
944  }
945  break;
946 
947  // Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy
948  case CURLE_HTTP2:
949  case CURLE_HTTP2_STREAM:
950  err = "Curl error " + str::numstring( code );
952  break;
953 
954  default:
955  err = "Curl error " + str::numstring( code );
956  break;
957  }
958 
959  // uhm, no 0 code but unknown curl exception
961  }
962  }
963  else
964  {
965  // actually the code is 0, nothing happened
966  }
967 }
968 
970 
971 bool MediaCurl::doGetDoesFileExist( const Pathname & filename ) const
972 {
973  DBG << filename.asString() << endl;
974 
975  if(!_url.isValid())
977 
978  if(_url.getHost().empty())
980 
981  Url url(getFileUrl(filename));
982 
983  DBG << "URL: " << url.asString() << endl;
984  // Use URL without options and without username and passwd
985  // (some proxies dislike them in the URL).
986  // Curl seems to need the just scheme, hostname and a path;
987  // the rest was already passed as curl options (in attachTo).
988  Url curlUrl( clearQueryString(url) );
989 
990  //
991  // See also Bug #154197 and ftp url definition in RFC 1738:
992  // The url "ftp://user@host/foo/bar/file" contains a path,
993  // that is relative to the user's home.
994  // The url "ftp://user@host//foo/bar/file" (or also with
995  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
996  // contains an absolute path.
997  //
998  _lastRedirect.clear();
999  std::string urlBuffer( curlUrl.asString());
1000  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1001  urlBuffer.c_str() );
1002  if ( ret != 0 ) {
1004  }
1005 
1006  // If no head requests allowed (?head_requests=no):
1007  // Instead of returning no data with NOBODY, we return
1008  // little data, that works with broken servers, and
1009  // works for ftp as well, because retrieving only headers
1010  // ftp will return always OK code ?
1011  // See http://curl.haxx.se/docs/knownbugs.html #58
1013  struct TempSetHeadRequest
1014  {
1015  TempSetHeadRequest( CURL * curl_r, bool doHttpHeadRequest_r )
1016  : _curl { curl_r }
1017  , _doHttpHeadRequest { doHttpHeadRequest_r }
1018  {
1019  if ( _doHttpHeadRequest ) {
1020  curl_easy_setopt( _curl, CURLOPT_NOBODY, 1L );
1021  } else {
1022  curl_easy_setopt( _curl, CURLOPT_RANGE, "0-1" );
1023  }
1024  }
1025  ~TempSetHeadRequest() {
1026  if ( _doHttpHeadRequest ) {
1027  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
1028  /* yes, this is why we never got to get NOBODY working before,
1029  because setting it changes this option too, and we also*
1030  need to reset it
1031  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
1032  */
1033  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
1034  } else {
1035  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
1036  }
1037  }
1038  private:
1039  CURL * _curl;
1040  bool _doHttpHeadRequest;
1041  } _guard( _curl, (_url.getScheme() == "http" || _url.getScheme() == "https") && _settings.headRequestsAllowed() );
1042 
1043 
1044  AutoFILE file { ::fopen( "/dev/null", "w" ) };
1045  if ( !file ) {
1046  ERR << "fopen failed for /dev/null" << endl;
1047  ZYPP_THROW(MediaWriteException("/dev/null"));
1048  }
1049 
1050  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, (*file) );
1051  if ( ret != 0 ) {
1053  }
1054 
1055  CURLcode ok = curl_easy_perform( _curl );
1056  MIL << "perform code: " << ok << " [ " << curl_easy_strerror(ok) << " ]" << endl;
1057 
1058  // as we are not having user interaction, the user can't cancel
1059  // the file existence checking, a callback or timeout return code
1060  // will be always a timeout.
1061  try {
1062  evaluateCurlCode( filename, ok, true /* timeout */);
1063  }
1064  catch ( const MediaFileNotFoundException &e ) {
1065  // if the file did not exist then we can return false
1066  return false;
1067  }
1068  catch ( const MediaException &e ) {
1069  // some error, we are not sure about file existence, rethrw
1070  ZYPP_RETHROW(e);
1071  }
1072  // exists
1073  return ( ok == CURLE_OK );
1074 }
1075 
1077 
1078 void MediaCurl::doGetFileCopy( const OnMediaLocation &srcFile , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1079 {
1080  Pathname dest = target.absolutename();
1081  if( assert_dir( dest.dirname() ) )
1082  {
1083  DBG << "assert_dir " << dest.dirname() << " failed" << endl;
1084  ZYPP_THROW( MediaSystemException(getFileUrl(srcFile.filename()), "System error on " + dest.dirname().asString()) );
1085  }
1086 
1087  ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) };
1088  AutoFILE file;
1089  {
1090  AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
1091  if( ! buf )
1092  {
1093  ERR << "out of memory for temp file name" << endl;
1094  ZYPP_THROW(MediaSystemException(getFileUrl(srcFile.filename()), "out of memory for temp file name"));
1095  }
1096 
1097  AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1098  if( tmp_fd == -1 )
1099  {
1100  ERR << "mkstemp failed for file '" << destNew << "'" << endl;
1101  ZYPP_THROW(MediaWriteException(destNew));
1102  }
1103  destNew = ManagedFile( (*buf), filesystem::unlink );
1104 
1105  file = ::fdopen( tmp_fd, "we" );
1106  if ( ! file )
1107  {
1108  ERR << "fopen failed for file '" << destNew << "'" << endl;
1109  ZYPP_THROW(MediaWriteException(destNew));
1110  }
1111  tmp_fd.resetDispose(); // don't close it here! ::fdopen moved ownership to file
1112  }
1113 
1114  DBG << "dest: " << dest << endl;
1115  DBG << "temp: " << destNew << endl;
1116 
1117  // set IFMODSINCE time condition (no download if not modified)
1118  if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
1119  {
1120  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1121  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
1122  }
1123  else
1124  {
1125  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1126  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1127  }
1128  try
1129  {
1130  doGetFileCopyFile( srcFile, dest, file, report, options);
1131  }
1132  catch (Exception &e)
1133  {
1134  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1135  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1136  ZYPP_RETHROW(e);
1137  }
1138 
1139  long httpReturnCode = 0;
1140  CURLcode infoRet = curl_easy_getinfo(_curl,
1141  CURLINFO_RESPONSE_CODE,
1142  &httpReturnCode);
1143  bool modified = true;
1144  if (infoRet == CURLE_OK)
1145  {
1146  DBG << "HTTP response: " + str::numstring(httpReturnCode);
1147  if ( httpReturnCode == 304
1148  || ( httpReturnCode == 213 && (_url.getScheme() == "ftp" || _url.getScheme() == "tftp") ) ) // not modified
1149  {
1150  DBG << " Not modified.";
1151  modified = false;
1152  }
1153  DBG << endl;
1154  }
1155  else
1156  {
1157  WAR << "Could not get the response code." << endl;
1158  }
1159 
1160  if (modified || infoRet != CURLE_OK)
1161  {
1162  // apply umask
1163  if ( ::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 ) ) )
1164  {
1165  ERR << "Failed to chmod file " << destNew << endl;
1166  }
1167 
1168  file.resetDispose(); // we're going to close it manually here
1169  if ( ::fclose( file ) )
1170  {
1171  ERR << "Fclose failed for file '" << destNew << "'" << endl;
1172  ZYPP_THROW(MediaWriteException(destNew));
1173  }
1174 
1175  // move the temp file into dest
1176  if ( rename( destNew, dest ) != 0 ) {
1177  ERR << "Rename failed" << endl;
1179  }
1180  destNew.resetDispose(); // no more need to unlink it
1181  }
1182 
1183  DBG << "done: " << PathInfo(dest) << endl;
1184 }
1185 
1187 
1188 void MediaCurl::doGetFileCopyFile( const OnMediaLocation & srcFile, const Pathname & dest, FILE *file, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1189 {
1190  DBG << srcFile.filename().asString() << endl;
1191 
1192  if(!_url.isValid())
1194 
1195  if(_url.getHost().empty())
1197 
1198  Url url(getFileUrl(srcFile.filename()));
1199 
1200  DBG << "URL: " << url.asString() << endl;
1201  // Use URL without options and without username and passwd
1202  // (some proxies dislike them in the URL).
1203  // Curl seems to need the just scheme, hostname and a path;
1204  // the rest was already passed as curl options (in attachTo).
1205  Url curlUrl( clearQueryString(url) );
1206 
1207  //
1208  // See also Bug #154197 and ftp url definition in RFC 1738:
1209  // The url "ftp://user@host/foo/bar/file" contains a path,
1210  // that is relative to the user's home.
1211  // The url "ftp://user@host//foo/bar/file" (or also with
1212  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1213  // contains an absolute path.
1214  //
1215  _lastRedirect.clear();
1216  std::string urlBuffer( curlUrl.asString());
1217  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1218  urlBuffer.c_str() );
1219  if ( ret != 0 ) {
1221  }
1222 
1223  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
1224  if ( ret != 0 ) {
1226  }
1227 
1228  // Set callback and perform.
1229  internal::ProgressData progressData(_curl, _settings.timeout(), url, srcFile.downloadSize(), &report);
1230  if (!(options & OPTION_NO_REPORT_START))
1231  report->start(url, dest);
1232  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
1233  WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1234  }
1235 
1236  ret = curl_easy_perform( _curl );
1237 #if CURLVERSION_AT_LEAST(7,19,4)
1238  // bnc#692260: If the client sends a request with an If-Modified-Since header
1239  // with a future date for the server, the server may respond 200 sending a
1240  // zero size file.
1241  // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
1242  if ( ftell(file) == 0 && ret == 0 )
1243  {
1244  long httpReturnCode = 33;
1245  if ( curl_easy_getinfo( _curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
1246  {
1247  long conditionUnmet = 33;
1248  if ( curl_easy_getinfo( _curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
1249  {
1250  WAR << "TIMECONDITION unmet - retry without." << endl;
1251  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1252  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1253  ret = curl_easy_perform( _curl );
1254  }
1255  }
1256  }
1257 #endif
1258 
1259  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
1260  WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1261  }
1262 
1263  if ( ret != 0 )
1264  {
1265  ERR << "curl error: " << ret << ": " << _curlError
1266  << ", temp file size " << ftell(file)
1267  << " bytes." << endl;
1268 
1269  // the timeout is determined by the progress data object
1270  // which holds whether the timeout was reached or not,
1271  // otherwise it would be a user cancel
1272  try {
1273 
1274  if ( progressData.fileSizeExceeded() )
1275  ZYPP_THROW(MediaFileSizeExceededException(url, progressData.expectedFileSize()));
1276 
1277  evaluateCurlCode( srcFile.filename(), ret, progressData.timeoutReached() );
1278  }
1279  catch ( const MediaException &e ) {
1280  // some error, we are not sure about file existence, rethrw
1281  ZYPP_RETHROW(e);
1282  }
1283  }
1284 }
1285 
1287 
1288 void MediaCurl::getDir( const Pathname & dirname, bool recurse_r ) const
1289 {
1290  filesystem::DirContent content;
1291  getDirInfo( content, dirname, /*dots*/false );
1292 
1293  for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
1294  Pathname filename = dirname + it->name;
1295  int res = 0;
1296 
1297  switch ( it->type ) {
1298  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
1299  case filesystem::FT_FILE:
1300  getFile( OnMediaLocation( filename ) );
1301  break;
1302  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
1303  if ( recurse_r ) {
1304  getDir( filename, recurse_r );
1305  } else {
1306  res = assert_dir( localPath( filename ) );
1307  if ( res ) {
1308  WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
1309  }
1310  }
1311  break;
1312  default:
1313  // don't provide devices, sockets, etc.
1314  break;
1315  }
1316  }
1317 }
1318 
1320 
1321 void MediaCurl::getDirInfo( std::list<std::string> & retlist,
1322  const Pathname & dirname, bool dots ) const
1323 {
1324  getDirectoryYast( retlist, dirname, dots );
1325 }
1326 
1328 
1330  const Pathname & dirname, bool dots ) const
1331 {
1332  getDirectoryYast( retlist, dirname, dots );
1333 }
1334 
1336 //
1337 int MediaCurl::aliveCallback( void *clientp, double /*dltotal*/, double dlnow, double /*ultotal*/, double /*ulnow*/ )
1338 {
1339  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1340  if( pdata )
1341  {
1342  // Do not propagate dltotal in alive callbacks. MultiCurl uses this to
1343  // prevent a percentage raise while downloading a metalink file. Download
1344  // activity however is indicated by propagating the download rate (via dlnow).
1345  pdata->updateStats( 0.0, dlnow );
1346  return pdata->reportProgress();
1347  }
1348  return 0;
1349 }
1350 
1351 int MediaCurl::progressCallback( void *clientp, double dltotal, double dlnow, double ultotal, double ulnow )
1352 {
1353  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1354  if( pdata )
1355  {
1356  // work around curl bug that gives us old data
1357  long httpReturnCode = 0;
1358  if ( curl_easy_getinfo( pdata->curl(), CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0 )
1359  return aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
1360 
1361  pdata->updateStats( dltotal, dlnow );
1362  return pdata->reportProgress();
1363  }
1364  return 0;
1365 }
1366 
1368 {
1369  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>(clientp);
1370  return pdata ? pdata->curl() : 0;
1371 }
1372 
1374 
1375 std::string MediaCurl::getAuthHint() const
1376 {
1377  long auth_info = CURLAUTH_NONE;
1378 
1379  CURLcode infoRet =
1380  curl_easy_getinfo(_curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1381 
1382  if(infoRet == CURLE_OK)
1383  {
1384  return CurlAuthData::auth_type_long2str(auth_info);
1385  }
1386 
1387  return "";
1388 }
1389 
1394 void MediaCurl::resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
1395 {
1396  internal::ProgressData *data = reinterpret_cast<internal::ProgressData *>(clientp);
1397  if ( data ) {
1398  data->expectedFileSize( expectedFileSize );
1399  }
1400 }
1401 
1403 
1404 bool MediaCurl::authenticate(const std::string & availAuthTypes, bool firstTry) const
1405 {
1407  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
1408  CurlAuthData_Ptr credentials;
1409 
1410  // get stored credentials
1411  AuthData_Ptr cmcred = cm.getCred(_url);
1412 
1413  if (cmcred && firstTry)
1414  {
1415  credentials.reset(new CurlAuthData(*cmcred));
1416  DBG << "got stored credentials:" << endl << *credentials << endl;
1417  }
1418  // if not found, ask user
1419  else
1420  {
1421 
1422  CurlAuthData_Ptr curlcred;
1423  curlcred.reset(new CurlAuthData());
1425 
1426  // preset the username if present in current url
1427  if (!_url.getUsername().empty() && firstTry)
1428  curlcred->setUsername(_url.getUsername());
1429  // if CM has found some credentials, preset the username from there
1430  else if (cmcred)
1431  curlcred->setUsername(cmcred->username());
1432 
1433  // indicate we have no good credentials from CM
1434  cmcred.reset();
1435 
1436  std::string prompt_msg = str::Format(_("Authentication required for '%s'")) % _url.asString();
1437 
1438  // set available authentication types from the exception
1439  // might be needed in prompt
1440  curlcred->setAuthType(availAuthTypes);
1441 
1442  // ask user
1443  if (auth_report->prompt(_url, prompt_msg, *curlcred))
1444  {
1445  DBG << "callback answer: retry" << endl
1446  << "CurlAuthData: " << *curlcred << endl;
1447 
1448  if (curlcred->valid())
1449  {
1450  credentials = curlcred;
1451  // if (credentials->username() != _url.getUsername())
1452  // _url.setUsername(credentials->username());
1460  }
1461  }
1462  else
1463  {
1464  DBG << "callback answer: cancel" << endl;
1465  }
1466  }
1467 
1468  // set username and password
1469  if (credentials)
1470  {
1471  // HACK, why is this const?
1472  const_cast<MediaCurl*>(this)->_settings.setUsername(credentials->username());
1473  const_cast<MediaCurl*>(this)->_settings.setPassword(credentials->password());
1474 
1475  // set username and password
1476  CURLcode ret = curl_easy_setopt(_curl, CURLOPT_USERPWD, _settings.userPassword().c_str());
1478 
1479  // set available authentication types from the exception
1480  if (credentials->authType() == CURLAUTH_NONE)
1481  credentials->setAuthType(availAuthTypes);
1482 
1483  // set auth type (seems this must be set _after_ setting the userpwd)
1484  if (credentials->authType() != CURLAUTH_NONE)
1485  {
1486  // FIXME: only overwrite if not empty?
1487  const_cast<MediaCurl*>(this)->_settings.setAuthType(credentials->authTypeAsString());
1488  ret = curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, credentials->authType());
1490  }
1491 
1492  if (!cmcred)
1493  {
1494  credentials->setUrl(_url);
1495  cm.addCred(*credentials);
1496  cm.save();
1497  }
1498 
1499  return true;
1500  }
1501 
1502  return false;
1503 }
1504 
1505 //need a out of line definiton, otherwise vtable is emitted for every translation unit
1507 
1508  } // namespace media
1509 } // namespace zypp
1510 //
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:533
long timeout() const
transfer timeout
void globalInitCurlOnce()
Definition: curlhelper.cc:64
Url getFileUrl(const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
virtual bool checkAttachPoint(const Pathname &apoint) const override
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
Definition: MediaCurl.cc:688
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:319
MediaCurlExceptionMayRetryInternaly(const Url &url_r, const std::string &err_r, const std::string &msg_r)
Definition: MediaCurl.cc:300
std::string targetDistribution() const
This is register.target attribute of the installed base product.
Definition: Target.cc:102
#define SET_OPTION_OFFT(opt, val)
Definition: MediaCurl.cc:330
#define MIL
Definition: Logger.h:96
const Pathname & clientCertificatePath() const
SSL client certificate file.
bool progress(int value_r, const Url &file_r, double dbps_avg_r=-1, double dbps_current_r=-1) override
Download progress.
Definition: MediaCurl.cc:83
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar" (trims)
#define _(MSG)
Definition: Gettext.h:37
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition: curlhelper.cc:137
ProgressData()
Ctor no range [0,0](0).
Definition: progressdata.h:174
const Pathname & path() const
Return current Pathname.
Definition: PathInfo.h:246
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
Describes a resource file located on a medium.
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:922
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
void checkProtocol(const Url &url) const
check the url is supported by the curl library
Definition: MediaCurl.cc:381
bool authenticate(const std::string &availAuthTypes, bool firstTry) const
Definition: MediaCurl.cc:1404
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl.h:31
void setPassword(const std::string &val_r)
sets the auth password
std::chrono::steady_clock::time_point TimePoint
Definition: MediaCurl.cc:51
ByteCount _expectedFileSize
Definition: MediaCurl.cc:167
virtual bool checkAttachPoint(const Pathname &apoint) const
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
const char * distributionFlavorHeader()
Definition: MediaCurl.cc:265
Store and operate with byte count.
Definition: ByteCount.h:30
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
to not add a IFMODSINCE header if target exists
Definition: MediaCurl.h:43
Holds transfer setting.
void save()
Saves any unsaved credentials added via addUserCred() or addGlobalCred() methods. ...
const std::string & authType() const
get the allowed authentication types
bool verifyHostEnabled() const
Whether to verify host for ssl.
const std::string & proxyUsername() const
proxy auth username
static int progressCallback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
Callback reporting download progress.
Definition: MediaCurl.cc:1351
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:173
int reportProgress() const
Definition: MediaCurl.cc:240
Url clearQueryString(const Url &url)
Definition: curlhelper.cc:367
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
void start(const Url &file_r, Pathname localfile_r) override
Definition: MediaCurl.cc:71
const char * anonymousIdHeader()
Definition: MediaCurl.cc:251
const char * c_str() const
String representation.
Definition: Pathname.h:110
void addHistory(const std::string &msg_r)
Add some message text to the history.
Definition: Exception.cc:140
bool isUseableAttachPoint(const Pathname &path, bool mtab=true) const
Ask media manager, if the specified path is already used as attach point or if there are another atta...
void setUsername(const std::string &val_r)
sets the auth username
Action problem(const Url &file_r, Error error_r, const std::string &description_r) override
Definition: MediaCurl.cc:102
virtual void getFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename) const override
Definition: MediaCurl.cc:730
static int aliveCallback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
Callback sending just an alive trigger to the UI, without stats (e.g.
Definition: MediaCurl.cc:1337
time_t _timeNow
Now.
Definition: MediaCurl.cc:173
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition: ManagedFile.h:27
virtual void setupEasy()
initializes the curl easy handle with the data from the url
Definition: MediaCurl.cc:406
bool timeoutReached() const
Definition: MediaCurl.cc:149
Convenient building of std::string with boost::format.
Definition: String.h:252
Structure holding values of curlrc options.
Definition: curlconfig.h:26
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:36
Edition * _value
Definition: SysContent.cc:311
AutoDispose<int> calling ::close
Definition: AutoDispose.h:301
std::string _currentCookieFile
Definition: MediaCurl.h:160
const std::string & password() const
auth password
virtual void getDir(const Pathname &dirname, bool recurse_r) const override
Call concrete handler to provide directory content (not recursive!) below attach point.
Definition: MediaCurl.cc:1288
virtual void getFile(const OnMediaLocation &file) const override
Call concrete handler to provide file below attach point.
Definition: MediaCurl.cc:721
#define ERR
Definition: Logger.h:98
const std::string & username() const
auth username
bool fileSizeExceeded() const
Definition: MediaCurl.cc:152
Pathname localPath(const Pathname &pathname) const
Files provided will be available at &#39;localPath(filename)&#39;.
const Headers & headers() const
returns a list of all added headers (trimmed)
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:45
static void setCookieFile(const Pathname &)
Definition: MediaCurl.cc:374
virtual void releaseFrom(const std::string &ejectDev) override
Call concrete handler to release the media.
Definition: MediaCurl.cc:714
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
static void resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
MediaMultiCurl needs to reset the expected filesize in case a metalink file is downloaded otherwise t...
Definition: MediaCurl.cc:1394
const std::string & hint() const
comma separated list of available authentication types
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:440
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: curlconfig.cc:24
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
Definition: PathInfo.cc:1202
double _dnlTotal
Bytes to download or 0 if unknown.
Definition: MediaCurl.cc:175
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:497
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:660
Bottleneck filtering all DownloadProgressReport issued from Media[Muli]Curl.
Definition: MediaCurl.cc:49
Url clearQueryString(const Url &url) const
Definition: MediaCurl.cc:369
void setAuthType(const std::string &val_r)
set the allowed authentication types
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:223
void finish(const Url &file_r, Error error_r, const std::string &reason_r) override
Definition: MediaCurl.cc:108
int unlink(const Pathname &path)
Like &#39;unlink&#39;.
Definition: PathInfo.cc:700
const Url _url
Url to handle.
Definition: MediaHandler.h:113
void setMediaSource(const MediaSourceRef &ref)
Set new media source reference.
const std::string & asString() const
String representation.
Definition: Pathname.h:91
int rename(const Pathname &oldpath, const Pathname &newpath)
Like &#39;rename&#39;.
Definition: PathInfo.cc:742
Just inherits Exception to separate media exceptions.
void evaluateCurlCode(const zypp::Pathname &filename, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition: MediaCurl.cc:823
std::string asUserHistory() const
A single (multiline) string composed of asUserString and historyAsString.
Definition: Exception.cc:91
const ByteCount & downloadSize() const
The size of the resource on the server.
void disconnect()
Use concrete handler to isconnect media.
long connectTimeout() const
connection timeout
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:124
do not send a start ProgressReport
Definition: MediaCurl.h:45
#define WAR
Definition: Logger.h:97
double _dnlLast
Bytes downloaded at period start.
Definition: MediaCurl.cc:176
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:518
void getDirectoryYast(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const
Retrieve and if available scan dirname/directory.yast.
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: curlhelper.cc:177
Receiver * _oldRec
std::string proxyuserpwd
Definition: curlconfig.h:49
bool isValid() const
Verifies the Url.
Definition: Url.cc:489
virtual bool doGetDoesFileExist(const Pathname &filename) const
Definition: MediaCurl.cc:971
const Pathname & clientKeyPath() const
SSL client key file.
void expectedFileSize(ByteCount newval_r)
Definition: MediaCurl.cc:158
shared_ptr< CurlAuthData > CurlAuthData_Ptr
Definition: curlauthdata.h:102
const Pathname & filename() const
The path to the resource on the medium.
virtual void attachTo(bool next=false) override
Call concrete handler to attach the media.
Definition: MediaCurl.cc:653
std::string numstring(char n, int w=0)
Definition: String.h:289
Common baseclass for MediaCurl and MediaNetwork.
virtual bool getDoesFileExist(const Pathname &filename) const override
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly, or user cancels the operation.
Definition: MediaCurl.cc:792
const char * agentString()
Definition: MediaCurl.cc:279
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:180
void doGetFileCopyFile(const OnMediaLocation &srcFile, const Pathname &dest, FILE *file, callback::SendReport< DownloadProgressReport > &report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1188
Media source internally used by MediaManager and MediaHandler.
Definition: MediaSource.h:36
static std::string auth_type_long2str(long auth_type)
Converts a long of ORed CURLAUTH_* identifiers into a string of comma separated list of authenticatio...
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
int _dnlPercent
Percent completed or 0 if _dnlTotal is unknown.
Definition: MediaCurl.cc:179
curl_slist * _customHeaders
Definition: MediaCurl.h:168
bool proxyEnabled() const
proxy is enabled
shared_ptr< AuthData > AuthData_Ptr
Definition: authdata.h:79
#define LIBZYPP_VERSION_STRING
Definition: APIConfig.h:15
int rmdir(const Pathname &path)
Like &#39;rmdir&#39;.
Definition: PathInfo.cc:366
#define SET_OPTION(opt, val)
Definition: MediaCurl.cc:323
Pathname absolutename() const
Return this path, adding a leading &#39;/&#39; if relative.
Definition: Pathname.h:139
Base class for Exception.
Definition: Exception.h:145
Pathname attachPoint() const
Return the currently used attach point.
std::string _lastRedirect
to log/report redirections
Definition: MediaCurl.h:163
Url url() const
Url used.
Definition: MediaHandler.h:503
double _drateLast
Download rate in last period.
Definition: MediaCurl.cc:182
std::string curlUnEscape(std::string text_r)
Definition: curlhelper.cc:360
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:588
time_t _timeStart
Start total stats.
Definition: MediaCurl.cc:170
virtual void getDirInfo(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const override
Call concrete handler to provide a content list of directory on media via retlist.
Definition: MediaCurl.cc:1321
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
Definition: curlhelper.cc:124
virtual void disconnectFrom() override
Definition: MediaCurl.cc:695
static CURL * progressCallback_getcurl(void *clientp)
Definition: MediaCurl.cc:1367
Typesafe passing of user data via callbacks.
Definition: UserData.h:38
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
double _dnlNow
Bytes downloaded now.
Definition: MediaCurl.cc:177
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3" (trims)
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:220
void report(const UserData &userData_r=UserData()) override
The most generic way of sending/receiving data.
Definition: MediaCurl.cc:67
AutoDispose<FILE*> calling ::fclose
Definition: AutoDispose.h:312
void updateStats(double dltotal=0.0, double dlnow=0.0)
Definition: MediaCurl.cc:197
static Pathname _cookieFile
Definition: MediaCurl.h:161
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: curlhelper.cc:325
double _drateTotal
Download rate so far.
Definition: MediaCurl.cc:181
std::string anonymousUniqueId() const
anonymous unique id
Definition: Target.cc:132
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:789
std::string userPassword() const
returns the user and password as a user:pass string
std::string getAuthHint() const
Return a comma separated list of available authentication methods supported by server.
Definition: MediaCurl.cc:1375
time_t _timeRcv
Start of no-data timeout.
Definition: MediaCurl.cc:172
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:21
Pathname createAttachPoint() const
Try to create a default / temporary attach point.
virtual void doGetFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename, callback::SendReport< DownloadProgressReport > &_report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1078
void addCred(const AuthData &cred)
Add new credentials with user callbacks.
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
Curl HTTP authentication data.
Definition: curlauthdata.h:22
std::string distributionFlavor() const
This is flavor attribute of the installed base product but does not require the target to be loaded a...
Definition: Target.cc:127
OptionalDownloadProgressReport(bool isOptional=false)
Definition: MediaCurl.cc:53
char _curlError[CURL_ERROR_SIZE]
Definition: MediaCurl.h:167
ByteCount expectedFileSize() const
Definition: MediaCurl.cc:155
const std::string & proxy() const
proxy host
MediaCurlException(const Url &url_r, const std::string &err_r, const std::string &msg_r)
bool optional() const
Whether this is an optional resource.
time_t _timeLast
Start last period(~1sec)
Definition: MediaCurl.cc:171
const char * c_str() const
Definition: IdStringType.h:105
TrueBool _guard
Definition: TargetImpl.cc:1596
bool userMayRWX() const
Definition: PathInfo.h:353
const std::string & userAgentString() const
user agent string (trimmed)
Url manipulation class.
Definition: Url.h:91
bool headRequestsAllowed() const
whether HEAD requests are allowed
#define DBG
Definition: Logger.h:95
Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy E.g. ...
Definition: MediaCurl.cc:297
zypp::callback::SendReport< zypp::media::DownloadProgressReport > * report
Definition: MediaCurl.cc:168
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:572