libzypp  17.31.8
MediaCurl.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
13 #include <iostream>
14 #include <chrono>
15 #include <list>
16 
17 #include <zypp/base/Logger.h>
18 #include <zypp/ExternalProgram.h>
19 #include <zypp/base/String.h>
20 #include <zypp/base/Gettext.h>
21 #include <zypp-core/parser/Sysconfig>
22 #include <zypp/base/Gettext.h>
23 
24 #include <zypp/media/MediaCurl.h>
25 #include <zypp-curl/ProxyInfo>
26 #include <zypp-curl/auth/CurlAuthData>
27 #include <zypp-media/auth/CredentialManager>
28 #include <zypp-curl/CurlConfig>
30 #include <zypp/Target.h>
31 #include <zypp/ZYppFactory.h>
32 #include <zypp/ZConfig.h>
33 
34 #include <cstdlib>
35 #include <sys/types.h>
36 #include <sys/stat.h>
37 #include <sys/mount.h>
38 #include <errno.h>
39 #include <dirent.h>
40 #include <unistd.h>
41 
42 using std::endl;
43 
44 namespace internal {
45  using namespace zypp;
49  struct OptionalDownloadProgressReport : public callback::ReceiveReport<media::DownloadProgressReport>
50  {
51  using TimePoint = std::chrono::steady_clock::time_point;
52 
53  OptionalDownloadProgressReport( bool isOptional=false )
54  : _oldRec { Distributor::instance().getReceiver() }
55  , _isOptional { isOptional }
56  { connect(); }
57 
59  { if ( _oldRec ) Distributor::instance().setReceiver( *_oldRec ); else Distributor::instance().noReceiver(); }
60 
61  void reportbegin() override
62  { if ( _oldRec ) _oldRec->reportbegin(); }
63 
64  void reportend() override
65  { if ( _oldRec ) _oldRec->reportend(); }
66 
67  void report( const UserData & userData_r = UserData() ) override
68  { if ( _oldRec ) _oldRec->report( userData_r ); }
69 
70 
71  void start( const Url & file_r, Pathname localfile_r ) override
72  {
73  if ( not _oldRec ) return;
74  if ( _isOptional ) {
75  // delay start until first data are received.
76  _startFile = file_r;
77  _startLocalfile = std::move(localfile_r);
78  return;
79  }
80  _oldRec->start( file_r, localfile_r );
81  }
82 
83  bool progress( int value_r, const Url & file_r, double dbps_avg_r = -1, double dbps_current_r = -1 ) override
84  {
85  if ( not _oldRec ) return true;
86  if ( notStarted() ) {
87  if ( not ( value_r || dbps_avg_r || dbps_current_r ) )
88  return true;
89  sendStart();
90  }
91 
92  //static constexpr std::chrono::milliseconds minfequency { 1000 }; only needed if we'd avoid sending reports without change
93  static constexpr std::chrono::milliseconds maxfequency { 100 };
94  TimePoint now { TimePoint::clock::now() };
95  TimePoint::duration elapsed { now - _lastProgressSent };
96  if ( elapsed < maxfequency )
97  return true; // continue
98  _lastProgressSent = now;
99  return _oldRec->progress( value_r, file_r, dbps_avg_r, dbps_current_r );
100  }
101 
102  Action problem( const Url & file_r, Error error_r, const std::string & description_r ) override
103  {
104  if ( not _oldRec || notStarted() ) return ABORT;
105  return _oldRec->problem( file_r, error_r, description_r );
106  }
107 
108  void finish( const Url & file_r, Error error_r, const std::string & reason_r ) override
109  {
110  if ( not _oldRec || notStarted() ) return;
111  _oldRec->finish( file_r, error_r, reason_r );
112  }
113 
114  private:
115  // _isOptional also indicates the delayed start
116  bool notStarted() const
117  { return _isOptional; }
118 
119  void sendStart()
120  {
121  if ( _isOptional ) {
122  // we know _oldRec is valid...
123  _oldRec->start( std::move(_startFile), std::move(_startLocalfile) );
124  _isOptional = false;
125  }
126  }
127 
128  private:
129  Receiver *const _oldRec;
132  Pathname _startLocalfile;
134  };
135 
137  {
138  ProgressData( CURL *curl, time_t timeout = 0, const zypp::Url & url = zypp::Url(),
139  zypp::ByteCount expectedFileSize_r = 0,
141 
142  void updateStats( double dltotal = 0.0, double dlnow = 0.0 );
143 
144  int reportProgress() const;
145 
146  CURL * curl()
147  { return _curl; }
148 
149  bool timeoutReached() const
150  { return _timeoutReached; }
151 
152  bool fileSizeExceeded() const
153  { return _fileSizeExceeded; }
154 
156  { return _expectedFileSize; }
157 
158  void expectedFileSize( ByteCount newval_r )
159  { _expectedFileSize = newval_r; }
160 
161  private:
162  CURL * _curl;
164  time_t _timeout;
169 
170  time_t _timeStart = 0;
171  time_t _timeLast = 0;
172  time_t _timeRcv = 0;
173  time_t _timeNow = 0;
174 
175  double _dnlTotal = 0.0;
176  double _dnlLast = 0.0;
177  double _dnlNow = 0.0;
178 
179  int _dnlPercent= 0;
180 
181  double _drateTotal= 0.0;
182  double _drateLast = 0.0;
183  };
184 
185 
186 
187  ProgressData::ProgressData(CURL *curl, time_t timeout, const Url &url, ByteCount expectedFileSize_r, zypp::callback::SendReport< zypp::media::DownloadProgressReport> *_report)
188  : _curl( curl )
189  , _url( url )
190  , _timeout( timeout )
191  , _timeoutReached( false )
192  , _fileSizeExceeded ( false )
193  , _expectedFileSize( expectedFileSize_r )
194  , report( _report )
195  {}
196 
197  void ProgressData::updateStats(double dltotal, double dlnow)
198  {
199  time_t now = _timeNow = time(0);
200 
201  // If called without args (0.0), recompute based on the last values seen
202  if ( dltotal && dltotal != _dnlTotal )
203  _dnlTotal = dltotal;
204 
205  if ( dlnow && dlnow != _dnlNow )
206  {
207  _timeRcv = now;
208  _dnlNow = dlnow;
209  }
210 
211  // init or reset if time jumps back
212  if ( !_timeStart || _timeStart > now )
213  _timeStart = _timeLast = _timeRcv = now;
214 
215  // timeout condition
216  if ( _timeout )
217  _timeoutReached = ( (now - _timeRcv) > _timeout );
218 
219  // check if the downloaded data is already bigger than what we expected
220  _fileSizeExceeded = _expectedFileSize > 0 && _expectedFileSize < static_cast<ByteCount::SizeType>(_dnlNow);
221 
222  // percentage:
223  if ( _dnlTotal )
224  _dnlPercent = int(_dnlNow * 100 / _dnlTotal);
225 
226  // download rates:
227  _drateTotal = _dnlNow / std::max( int(now - _timeStart), 1 );
228 
229  if ( _timeLast < now )
230  {
231  _drateLast = (_dnlNow - _dnlLast) / int(now - _timeLast);
232  // start new period
233  _timeLast = now;
234  _dnlLast = _dnlNow;
235  }
236  else if ( _timeStart == _timeLast )
238  }
239 
241  {
242  if ( _fileSizeExceeded )
243  return 1;
244  if ( _timeoutReached )
245  return 1; // no-data timeout
246  if ( report && !(*report)->progress( _dnlPercent, _url, _drateTotal, _drateLast ) )
247  return 1; // user requested abort
248  return 0;
249  }
250 
251  const char * anonymousIdHeader()
252  {
253  // we need to add the release and identifier to the
254  // agent string.
255  // The target could be not initialized, and then this information
256  // is guessed.
257  static const std::string _value(
259  "X-ZYpp-AnonymousId: %s",
260  Target::anonymousUniqueId( Pathname()/*guess root*/ ).c_str() ) )
261  );
262  return _value.c_str();
263  }
264 
266  {
267  // we need to add the release and identifier to the
268  // agent string.
269  // The target could be not initialized, and then this information
270  // is guessed.
271  static const std::string _value(
273  "X-ZYpp-DistributionFlavor: %s",
274  Target::distributionFlavor( Pathname()/*guess root*/ ).c_str() ) )
275  );
276  return _value.c_str();
277  }
278 
279  const char * agentString()
280  {
281  // we need to add the release and identifier to the
282  // agent string.
283  // The target could be not initialized, and then this information
284  // is guessed.
285  static const std::string _value(
286  str::form(
287  "ZYpp " LIBZYPP_VERSION_STRING " (curl %s) %s"
288  , curl_version_info(CURLVERSION_NOW)->version
289  , Target::targetDistribution( Pathname()/*guess root*/ ).c_str()
290  )
291  );
292  return _value.c_str();
293  }
294 
298  {
299  public:
301  const std::string & err_r,
302  const std::string & msg_r )
303  : media::MediaCurlException( url_r, err_r, msg_r )
304  {}
305  //~MediaCurlExceptionMayRetryInternaly() noexcept {}
306  };
307 
308 }
309 
310 
311 using namespace internal;
312 using namespace zypp::base;
313 
314 namespace zypp {
315 
316  namespace media {
317 
318 Pathname MediaCurl::_cookieFile = "/var/lib/YaST2/cookies";
319 
320 // we use this define to unbloat code as this C setting option
321 // and catching exception is done frequently.
323 #define SET_OPTION(opt,val) do { \
324  ret = curl_easy_setopt ( _curl, opt, val ); \
325  if ( ret != 0) { \
326  ZYPP_THROW(MediaCurlSetOptException(_url, _curlError)); \
327  } \
328  } while ( false )
329 
330 #define SET_OPTION_OFFT(opt,val) SET_OPTION(opt,(curl_off_t)val)
331 #define SET_OPTION_LONG(opt,val) SET_OPTION(opt,(long)val)
332 #define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val)
333 
334 MediaCurl::MediaCurl( const Url & url_r,
335  const Pathname & attach_point_hint_r )
336  : MediaNetworkCommonHandler( url_r, attach_point_hint_r,
337  "/", // urlpath at attachpoint
338  true ), // does_download
339  _curl( NULL ),
340  _customHeaders(0L)
341 {
342  _curlError[0] = '\0';
343  _curlDebug = 0L;
344 
345  MIL << "MediaCurl::MediaCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl;
346 
348 
349  if( !attachPoint().empty())
350  {
351  PathInfo ainfo(attachPoint());
352  Pathname apath(attachPoint() + "XXXXXX");
353  char *atemp = ::strdup( apath.asString().c_str());
354  char *atest = NULL;
355  if( !ainfo.isDir() || !ainfo.userMayRWX() ||
356  atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
357  {
358  WAR << "attach point " << ainfo.path()
359  << " is not useable for " << url_r.getScheme() << endl;
360  setAttachPoint("", true);
361  }
362  else if( atest != NULL)
363  ::rmdir(atest);
364 
365  if( atemp != NULL)
366  ::free(atemp);
367  }
368 }
369 
371 {
373 }
374 
375 void MediaCurl::setCookieFile( const Pathname &fileName )
376 {
377  _cookieFile = fileName;
378 }
379 
381 
382 void MediaCurl::checkProtocol(const Url &url) const
383 {
384  curl_version_info_data *curl_info = NULL;
385  curl_info = curl_version_info(CURLVERSION_NOW);
386  // curl_info does not need any free (is static)
387  if (curl_info->protocols)
388  {
389  const char * const *proto;
390  std::string scheme( url.getScheme());
391  bool found = false;
392  for(proto=curl_info->protocols; !found && *proto; ++proto)
393  {
394  if( scheme == std::string((const char *)*proto))
395  found = true;
396  }
397  if( !found)
398  {
399  std::string msg("Unsupported protocol '");
400  msg += scheme;
401  msg += "'";
403  }
404  }
405 }
406 
408 {
409  {
411  if( _curlDebug > 0)
412  {
413  curl_easy_setopt( _curl, CURLOPT_VERBOSE, 1L);
414  curl_easy_setopt( _curl, CURLOPT_DEBUGFUNCTION, log_curl);
415  curl_easy_setopt( _curl, CURLOPT_DEBUGDATA, &_curlDebug);
416  }
417  }
418 
419  curl_easy_setopt(_curl, CURLOPT_HEADERFUNCTION, log_redirects_curl);
420  curl_easy_setopt(_curl, CURLOPT_HEADERDATA, &_lastRedirect);
421  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_ERRORBUFFER, _curlError );
422  if ( ret != 0 ) {
423  ZYPP_THROW(MediaCurlSetOptException(_url, "Error setting error buffer"));
424  }
425 
426  SET_OPTION(CURLOPT_FAILONERROR, 1L);
427  SET_OPTION(CURLOPT_NOSIGNAL, 1L);
428 
429  // create non persistant settings
430  // so that we don't add headers twice
431  TransferSettings vol_settings(_settings);
432 
433  // add custom headers for download.opensuse.org (bsc#955801)
434  if ( _url.getHost() == "download.opensuse.org" )
435  {
436  vol_settings.addHeader(anonymousIdHeader());
437  vol_settings.addHeader(distributionFlavorHeader());
438  }
439  vol_settings.addHeader("Pragma:");
440 
441  _settings.setTimeout(ZConfig::instance().download_transfer_timeout());
443 
445 
446  // fill some settings from url query parameters
447  try
448  {
450  }
451  catch ( const MediaException &e )
452  {
453  disconnectFrom();
454  ZYPP_RETHROW(e);
455  }
456  // if the proxy was not set (or explicitly unset) by url, then look...
457  if ( _settings.proxy().empty() )
458  {
459  // ...at the system proxy settings
461  }
462 
464  switch ( env::ZYPP_MEDIA_CURL_IPRESOLVE() )
465  {
466  case 4: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); break;
467  case 6: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); break;
468  }
469 
473  SET_OPTION(CURLOPT_CONNECTTIMEOUT, _settings.connectTimeout());
474  // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
475  // just in case curl does not trigger its progress callback frequently
476  // enough.
477  if ( _settings.timeout() )
478  {
479  SET_OPTION(CURLOPT_TIMEOUT, 3600L);
480  }
481 
482  // follow any Location: header that the server sends as part of
483  // an HTTP header (#113275)
484  SET_OPTION(CURLOPT_FOLLOWLOCATION, 1L);
485  // 3 redirects seem to be too few in some cases (bnc #465532)
486  SET_OPTION(CURLOPT_MAXREDIRS, 6L);
487 
488  if ( _url.getScheme() == "https" )
489  {
490 #if CURLVERSION_AT_LEAST(7,19,4)
491  // restrict following of redirections from https to https only
492  if ( _url.getHost() == "download.opensuse.org" )
493  SET_OPTION( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS );
494  else
495  SET_OPTION( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
496 #endif
497 
500  {
502  }
503 
505  {
506  SET_OPTION(CURLOPT_SSLCERT, _settings.clientCertificatePath().c_str());
507  }
508  if( ! _settings.clientKeyPath().empty() )
509  {
510  SET_OPTION(CURLOPT_SSLKEY, _settings.clientKeyPath().c_str());
511  }
512 
513 #ifdef CURLSSLOPT_ALLOW_BEAST
514  // see bnc#779177
515  ret = curl_easy_setopt( _curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
516  if ( ret != 0 ) {
517  disconnectFrom();
519  }
520 #endif
521  SET_OPTION(CURLOPT_SSL_VERIFYPEER, _settings.verifyPeerEnabled() ? 1L : 0L);
522  SET_OPTION(CURLOPT_SSL_VERIFYHOST, _settings.verifyHostEnabled() ? 2L : 0L);
523  // bnc#903405 - POODLE: libzypp should only talk TLS
524  SET_OPTION(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
525  }
526 
527  SET_OPTION(CURLOPT_USERAGENT, _settings.userAgentString().c_str() );
528 
529  /* Fixes bsc#1174011 "auth=basic ignored in some cases"
530  * We should proactively add the password to the request if basic auth is configured
531  * and a password is available in the credentials but not in the URL.
532  *
533  * We will be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
534  * and ask the server first about the auth method
535  */
536  if ( _settings.authType() == "basic"
537  && _settings.username().size()
538  && !_settings.password().size() ) {
539 
540  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
541  const auto cred = cm.getCred( _url );
542  if ( cred && cred->valid() ) {
543  if ( !_settings.username().size() )
544  _settings.setUsername(cred->username());
545  _settings.setPassword(cred->password());
546  }
547  }
548 
549  /*---------------------------------------------------------------*
550  CURLOPT_USERPWD: [user name]:[password]
551 
552  Url::username/password -> CURLOPT_USERPWD
553  If not provided, anonymous FTP identification
554  *---------------------------------------------------------------*/
555 
556  if ( _settings.userPassword().size() )
557  {
558  SET_OPTION(CURLOPT_USERPWD, _settings.userPassword().c_str());
559  std::string use_auth = _settings.authType();
560  if (use_auth.empty())
561  use_auth = "digest,basic"; // our default
562  long auth = CurlAuthData::auth_type_str2long(use_auth);
563  if( auth != CURLAUTH_NONE)
564  {
565  DBG << "Enabling HTTP authentication methods: " << use_auth
566  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
567  SET_OPTION(CURLOPT_HTTPAUTH, auth);
568  }
569  }
570 
571  if ( _settings.proxyEnabled() && ! _settings.proxy().empty() )
572  {
573  DBG << "Proxy: '" << _settings.proxy() << "'" << endl;
574  SET_OPTION(CURLOPT_PROXY, _settings.proxy().c_str());
575  SET_OPTION(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
576  /*---------------------------------------------------------------*
577  * CURLOPT_PROXYUSERPWD: [user name]:[password]
578  *
579  * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
580  * If not provided, $HOME/.curlrc is evaluated
581  *---------------------------------------------------------------*/
582 
583  std::string proxyuserpwd = _settings.proxyUserPassword();
584 
585  if ( proxyuserpwd.empty() )
586  {
587  CurlConfig curlconf;
588  CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
589  if ( curlconf.proxyuserpwd.empty() )
590  DBG << "Proxy: ~/.curlrc does not contain the proxy-user option" << endl;
591  else
592  {
593  proxyuserpwd = curlconf.proxyuserpwd;
594  DBG << "Proxy: using proxy-user from ~/.curlrc" << endl;
595  }
596  }
597  else
598  {
599  DBG << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << endl;
600  }
601 
602  if ( ! proxyuserpwd.empty() )
603  {
604  SET_OPTION(CURLOPT_PROXYUSERPWD, curlUnEscape( proxyuserpwd ).c_str());
605  }
606  }
607 #if CURLVERSION_AT_LEAST(7,19,4)
608  else if ( _settings.proxy() == EXPLICITLY_NO_PROXY )
609  {
610  // Explicitly disabled in URL (see fillSettingsFromUrl()).
611  // This should also prevent libcurl from looking into the environment.
612  DBG << "Proxy: explicitly NOPROXY" << endl;
613  SET_OPTION(CURLOPT_NOPROXY, "*");
614  }
615 #endif
616  else
617  {
618  DBG << "Proxy: not explicitly set" << endl;
619  DBG << "Proxy: libcurl may look into the environment" << endl;
620  }
621 
623  if ( _settings.minDownloadSpeed() != 0 )
624  {
625  SET_OPTION(CURLOPT_LOW_SPEED_LIMIT, _settings.minDownloadSpeed());
626  // default to 10 seconds at low speed
627  SET_OPTION(CURLOPT_LOW_SPEED_TIME, 60L);
628  }
629 
630 #if CURLVERSION_AT_LEAST(7,15,5)
631  if ( _settings.maxDownloadSpeed() != 0 )
632  SET_OPTION_OFFT(CURLOPT_MAX_RECV_SPEED_LARGE, _settings.maxDownloadSpeed());
633 #endif
634 
635  /*---------------------------------------------------------------*
636  *---------------------------------------------------------------*/
637 
640  if ( str::strToBool( _url.getQueryParam( "cookies" ), true ) )
641  SET_OPTION(CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
642  else
643  MIL << "No cookies requested" << endl;
644  SET_OPTION(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
645  SET_OPTION(CURLOPT_PROGRESSFUNCTION, &progressCallback );
646  SET_OPTION(CURLOPT_NOPROGRESS, 0L);
647 
648 #if CURLVERSION_AT_LEAST(7,18,0)
649  // bnc #306272
650  SET_OPTION(CURLOPT_PROXY_TRANSFER_MODE, 1L );
651 #endif
652  // append settings custom headers to curl
653  for ( const auto &header : vol_settings.headers() )
654  {
655  // MIL << "HEADER " << *it << std::endl;
656 
657  _customHeaders = curl_slist_append(_customHeaders, header.c_str());
658  if ( !_customHeaders )
660  }
661 
662  SET_OPTION(CURLOPT_HTTPHEADER, _customHeaders);
663 }
664 
666 
667 
668 void MediaCurl::attachTo (bool next)
669 {
670  if ( next )
672 
673  if ( !_url.isValid() )
675 
678  {
680  }
681 
682  disconnectFrom(); // clean _curl if needed
683  _curl = curl_easy_init();
684  if ( !_curl ) {
686  }
687  try
688  {
689  setupEasy();
690  }
691  catch (Exception & ex)
692  {
693  disconnectFrom();
694  ZYPP_RETHROW(ex);
695  }
696 
697  // FIXME: need a derived class to propelly compare url's
699  setMediaSource(media);
700 }
701 
702 bool
704 {
705  return MediaHandler::checkAttachPoint( apoint, true, true);
706 }
707 
709 
711 {
712  if ( _customHeaders )
713  {
714  curl_slist_free_all(_customHeaders);
715  _customHeaders = 0L;
716  }
717 
718  if ( _curl )
719  {
720  // bsc#1201092: Strange but within global_dtors we may exceptions here.
721  try { curl_easy_cleanup( _curl ); }
722  catch (...) { ; }
723  _curl = NULL;
724  }
725 }
726 
728 
729 void MediaCurl::releaseFrom( const std::string & ejectDev )
730 {
731  disconnect();
732 }
733 
735 
736 void MediaCurl::getFile( const OnMediaLocation &file ) const
737 {
738  // Use absolute file name to prevent access of files outside of the
739  // hierarchy below the attach point.
740  getFileCopy( file, localPath(file.filename()).absolutename() );
741 }
742 
744 
745 void MediaCurl::getFileCopy( const OnMediaLocation & srcFile , const Pathname & target ) const
746 {
747 
748  const auto &filename = srcFile.filename();
749 
750  // Optional files will send no report until data are actually received (we know it exists).
751  OptionalDownloadProgressReport reportfilter( srcFile.optional() );
753 
754  Url fileurl(getFileUrl(filename));
755 
756  bool retry = false;
757  unsigned internalTry = 0;
758  static constexpr unsigned maxInternalTry = 3;
759 
760  do
761  {
762  retry = false;
763  try
764  {
765  doGetFileCopy( srcFile, target, report );
766  }
767  // retry with proper authentication data
768  catch (MediaUnauthorizedException & ex_r)
769  {
770  if(authenticate(ex_r.hint(), !retry))
771  retry = true;
772  else
773  {
774  report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
775  ZYPP_RETHROW(ex_r);
776  }
777  }
778  // unexpected exception
779  catch (MediaException & excpt_r)
780  {
781  if ( typeid(excpt_r) == typeid( MediaCurlExceptionMayRetryInternaly ) ) {
782  ++internalTry;
783  if ( internalTry < maxInternalTry ) {
784  // just report (NO_ERROR); no interactive request to the user
785  report->problem(fileurl, media::DownloadProgressReport::NO_ERROR, excpt_r.asUserHistory()+_("Will try again..."));
786  retry = true;
787  continue;
788  }
789  excpt_r.addHistory( str::Format(_("Giving up after %1% attempts.")) % maxInternalTry );
790  }
791 
793  if( typeid(excpt_r) == typeid( media::MediaFileNotFoundException ) ||
794  typeid(excpt_r) == typeid( media::MediaNotAFileException ) )
795  {
797  }
798  report->finish(fileurl, reason, excpt_r.asUserHistory());
799  ZYPP_RETHROW(excpt_r);
800  }
801  }
802  while (retry);
803 
804  report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
805 }
806 
808 
809 bool MediaCurl::getDoesFileExist( const Pathname & filename ) const
810 {
811  bool retry = false;
812 
813  do
814  {
815  try
816  {
817  return doGetDoesFileExist( filename );
818  }
819  // authentication problem, retry with proper authentication data
820  catch (MediaUnauthorizedException & ex_r)
821  {
822  if(authenticate(ex_r.hint(), !retry))
823  retry = true;
824  else
825  ZYPP_RETHROW(ex_r);
826  }
827  // unexpected exception
828  catch (MediaException & excpt_r)
829  {
830  ZYPP_RETHROW(excpt_r);
831  }
832  }
833  while (retry);
834 
835  return false;
836 }
837 
839 
841  CURLcode code,
842  bool timeout_reached) const
843 {
844  if ( code != 0 )
845  {
846  Url url;
847  if (filename.empty())
848  url = _url;
849  else
850  url = getFileUrl(filename);
851 
852  std::string err;
853  {
854  switch ( code )
855  {
856  case CURLE_UNSUPPORTED_PROTOCOL:
857  err = " Unsupported protocol";
858  if ( !_lastRedirect.empty() )
859  {
860  err += " or redirect (";
861  err += _lastRedirect;
862  err += ")";
863  }
864  break;
865  case CURLE_URL_MALFORMAT:
866  case CURLE_URL_MALFORMAT_USER:
867  err = " Bad URL";
868  break;
869  case CURLE_LOGIN_DENIED:
870  ZYPP_THROW(
871  MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
872  break;
873  case CURLE_HTTP_RETURNED_ERROR:
874  {
875  long httpReturnCode = 0;
876  CURLcode infoRet = curl_easy_getinfo( _curl,
877  CURLINFO_RESPONSE_CODE,
878  &httpReturnCode );
879  if ( infoRet == CURLE_OK )
880  {
881  std::string msg = "HTTP response: " + str::numstring( httpReturnCode );
882  switch ( httpReturnCode )
883  {
884  case 401:
885  {
886  std::string auth_hint = getAuthHint();
887 
888  DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
889  DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
890 
892  url, "Login failed.", _curlError, auth_hint
893  ));
894  }
895 
896  case 502: // bad gateway (bnc #1070851)
897  case 503: // service temporarily unavailable (bnc #462545)
899  case 504: // gateway timeout
901  case 403:
902  {
903  std::string msg403;
904  if ( url.getHost().find(".suse.com") != std::string::npos )
905  msg403 = _("Visit the SUSE Customer Center to check whether your registration is valid and has not expired.");
906  else if (url.asString().find("novell.com") != std::string::npos)
907  msg403 = _("Visit the Novell Customer Center to check whether your registration is valid and has not expired.");
909  }
910  case 404:
911  case 410:
913  }
914 
915  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
917  }
918  else
919  {
920  std::string msg = "Unable to retrieve HTTP response:";
921  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
923  }
924  }
925  break;
926  case CURLE_FTP_COULDNT_RETR_FILE:
927 #if CURLVERSION_AT_LEAST(7,16,0)
928  case CURLE_REMOTE_FILE_NOT_FOUND:
929 #endif
930  case CURLE_FTP_ACCESS_DENIED:
931  case CURLE_TFTP_NOTFOUND:
932  err = "File not found";
934  break;
935  case CURLE_BAD_PASSWORD_ENTERED:
936  case CURLE_FTP_USER_PASSWORD_INCORRECT:
937  err = "Login failed";
938  break;
939  case CURLE_COULDNT_RESOLVE_PROXY:
940  case CURLE_COULDNT_RESOLVE_HOST:
941  case CURLE_COULDNT_CONNECT:
942  case CURLE_FTP_CANT_GET_HOST:
943  err = "Connection failed";
944  break;
945  case CURLE_WRITE_ERROR:
946  err = "Write error";
947  break;
948  case CURLE_PARTIAL_FILE:
949  case CURLE_OPERATION_TIMEDOUT:
950  timeout_reached = true; // fall though to TimeoutException
951  // fall though...
952  case CURLE_ABORTED_BY_CALLBACK:
953  if( timeout_reached )
954  {
955  err = "Timeout reached";
957  }
958  else
959  {
960  err = "User abort";
961  }
962  break;
963 
964  // Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy
965  case CURLE_HTTP2:
966  case CURLE_HTTP2_STREAM:
967  err = "Curl error " + str::numstring( code );
969  break;
970 
971  default:
972  err = "Curl error " + str::numstring( code );
973  break;
974  }
975 
976  // uhm, no 0 code but unknown curl exception
978  }
979  }
980  else
981  {
982  // actually the code is 0, nothing happened
983  }
984 }
985 
987 
988 bool MediaCurl::doGetDoesFileExist( const Pathname & filename ) const
989 {
990  DBG << filename.asString() << endl;
991 
992  if(!_url.isValid())
994 
995  if(_url.getHost().empty())
997 
998  Url url(getFileUrl(filename));
999 
1000  DBG << "URL: " << url.asString() << endl;
1001  // Use URL without options and without username and passwd
1002  // (some proxies dislike them in the URL).
1003  // Curl seems to need the just scheme, hostname and a path;
1004  // the rest was already passed as curl options (in attachTo).
1005  Url curlUrl( clearQueryString(url) );
1006 
1007  //
1008  // See also Bug #154197 and ftp url definition in RFC 1738:
1009  // The url "ftp://user@host/foo/bar/file" contains a path,
1010  // that is relative to the user's home.
1011  // The url "ftp://user@host//foo/bar/file" (or also with
1012  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1013  // contains an absolute path.
1014  //
1015  _lastRedirect.clear();
1016  std::string urlBuffer( curlUrl.asString());
1017  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1018  urlBuffer.c_str() );
1019  if ( ret != 0 ) {
1021  }
1022 
1023  // If no head requests allowed (?head_requests=no):
1024  // Instead of returning no data with NOBODY, we return
1025  // little data, that works with broken servers, and
1026  // works for ftp as well, because retrieving only headers
1027  // ftp will return always OK code ?
1028  // See http://curl.haxx.se/docs/knownbugs.html #58
1030  struct TempSetHeadRequest
1031  {
1032  TempSetHeadRequest( CURL * curl_r, bool doHttpHeadRequest_r )
1033  : _curl { curl_r }
1034  , _doHttpHeadRequest { doHttpHeadRequest_r }
1035  {
1036  if ( _doHttpHeadRequest ) {
1037  curl_easy_setopt( _curl, CURLOPT_NOBODY, 1L );
1038  } else {
1039  curl_easy_setopt( _curl, CURLOPT_RANGE, "0-1" );
1040  }
1041  }
1042  ~TempSetHeadRequest() {
1043  if ( _doHttpHeadRequest ) {
1044  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
1045  /* yes, this is why we never got to get NOBODY working before,
1046  because setting it changes this option too, and we also*
1047  need to reset it
1048  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
1049  */
1050  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
1051  } else {
1052  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
1053  }
1054  }
1055  private:
1056  CURL * _curl;
1057  bool _doHttpHeadRequest;
1058  } _guard( _curl, (_url.getScheme() == "http" || _url.getScheme() == "https") && _settings.headRequestsAllowed() );
1059 
1060 
1061  AutoFILE file { ::fopen( "/dev/null", "w" ) };
1062  if ( !file ) {
1063  ERR << "fopen failed for /dev/null" << endl;
1064  ZYPP_THROW(MediaWriteException("/dev/null"));
1065  }
1066 
1067  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, (*file) );
1068  if ( ret != 0 ) {
1070  }
1071 
1072  CURLcode ok = curl_easy_perform( _curl );
1073  MIL << "perform code: " << ok << " [ " << curl_easy_strerror(ok) << " ]" << endl;
1074 
1075  // as we are not having user interaction, the user can't cancel
1076  // the file existence checking, a callback or timeout return code
1077  // will be always a timeout.
1078  try {
1079  evaluateCurlCode( filename, ok, true /* timeout */);
1080  }
1081  catch ( const MediaFileNotFoundException &e ) {
1082  // if the file did not exist then we can return false
1083  return false;
1084  }
1085  catch ( const MediaException &e ) {
1086  // some error, we are not sure about file existence, rethrw
1087  ZYPP_RETHROW(e);
1088  }
1089  // exists
1090  return ( ok == CURLE_OK );
1091 }
1092 
1094 
1095 
1096 #if DETECT_DIR_INDEX
1097 bool MediaCurl::detectDirIndex() const
1098 {
1099  if(_url.getScheme() != "http" && _url.getScheme() != "https")
1100  return false;
1101  //
1102  // try to check the effective url and set the not_a_file flag
1103  // if the url path ends with a "/", what usually means, that
1104  // we've received a directory index (index.html content).
1105  //
1106  // Note: This may be dangerous and break file retrieving in
1107  // case of some server redirections ... ?
1108  //
1109  bool not_a_file = false;
1110  char *ptr = NULL;
1111  CURLcode ret = curl_easy_getinfo( _curl,
1112  CURLINFO_EFFECTIVE_URL,
1113  &ptr);
1114  if ( ret == CURLE_OK && ptr != NULL)
1115  {
1116  try
1117  {
1118  Url eurl( ptr);
1119  std::string path( eurl.getPathName());
1120  if( !path.empty() && path != "/" && *path.rbegin() == '/')
1121  {
1122  DBG << "Effective url ("
1123  << eurl
1124  << ") seems to provide the index of a directory"
1125  << endl;
1126  not_a_file = true;
1127  }
1128  }
1129  catch( ... )
1130  {}
1131  }
1132  return not_a_file;
1133 }
1134 #endif
1135 
1137 
1138 void MediaCurl::doGetFileCopy( const OnMediaLocation &srcFile , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1139 {
1140  Pathname dest = target.absolutename();
1141  if( assert_dir( dest.dirname() ) )
1142  {
1143  DBG << "assert_dir " << dest.dirname() << " failed" << endl;
1144  ZYPP_THROW( MediaSystemException(getFileUrl(srcFile.filename()), "System error on " + dest.dirname().asString()) );
1145  }
1146 
1147  ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) };
1148  AutoFILE file;
1149  {
1150  AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
1151  if( ! buf )
1152  {
1153  ERR << "out of memory for temp file name" << endl;
1154  ZYPP_THROW(MediaSystemException(getFileUrl(srcFile.filename()), "out of memory for temp file name"));
1155  }
1156 
1157  AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1158  if( tmp_fd == -1 )
1159  {
1160  ERR << "mkstemp failed for file '" << destNew << "'" << endl;
1161  ZYPP_THROW(MediaWriteException(destNew));
1162  }
1163  destNew = ManagedFile( (*buf), filesystem::unlink );
1164 
1165  file = ::fdopen( tmp_fd, "we" );
1166  if ( ! file )
1167  {
1168  ERR << "fopen failed for file '" << destNew << "'" << endl;
1169  ZYPP_THROW(MediaWriteException(destNew));
1170  }
1171  tmp_fd.resetDispose(); // don't close it here! ::fdopen moved ownership to file
1172  }
1173 
1174  DBG << "dest: " << dest << endl;
1175  DBG << "temp: " << destNew << endl;
1176 
1177  // set IFMODSINCE time condition (no download if not modified)
1178  if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
1179  {
1180  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1181  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
1182  }
1183  else
1184  {
1185  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1186  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1187  }
1188  try
1189  {
1190  doGetFileCopyFile( srcFile, dest, file, report, options);
1191  }
1192  catch (Exception &e)
1193  {
1194  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1195  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1196  ZYPP_RETHROW(e);
1197  }
1198 
1199  long httpReturnCode = 0;
1200  CURLcode infoRet = curl_easy_getinfo(_curl,
1201  CURLINFO_RESPONSE_CODE,
1202  &httpReturnCode);
1203  bool modified = true;
1204  if (infoRet == CURLE_OK)
1205  {
1206  DBG << "HTTP response: " + str::numstring(httpReturnCode);
1207  if ( httpReturnCode == 304
1208  || ( httpReturnCode == 213 && (_url.getScheme() == "ftp" || _url.getScheme() == "tftp") ) ) // not modified
1209  {
1210  DBG << " Not modified.";
1211  modified = false;
1212  }
1213  DBG << endl;
1214  }
1215  else
1216  {
1217  WAR << "Could not get the response code." << endl;
1218  }
1219 
1220  if (modified || infoRet != CURLE_OK)
1221  {
1222  // apply umask
1223  if ( ::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 ) ) )
1224  {
1225  ERR << "Failed to chmod file " << destNew << endl;
1226  }
1227 
1228  file.resetDispose(); // we're going to close it manually here
1229  if ( ::fclose( file ) )
1230  {
1231  ERR << "Fclose failed for file '" << destNew << "'" << endl;
1232  ZYPP_THROW(MediaWriteException(destNew));
1233  }
1234 
1235  // move the temp file into dest
1236  if ( rename( destNew, dest ) != 0 ) {
1237  ERR << "Rename failed" << endl;
1239  }
1240  destNew.resetDispose(); // no more need to unlink it
1241  }
1242 
1243  DBG << "done: " << PathInfo(dest) << endl;
1244 }
1245 
1247 
1248 void MediaCurl::doGetFileCopyFile( const OnMediaLocation & srcFile, const Pathname & dest, FILE *file, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1249 {
1250  DBG << srcFile.filename().asString() << endl;
1251 
1252  if(!_url.isValid())
1254 
1255  if(_url.getHost().empty())
1257 
1258  Url url(getFileUrl(srcFile.filename()));
1259 
1260  DBG << "URL: " << url.asString() << endl;
1261  // Use URL without options and without username and passwd
1262  // (some proxies dislike them in the URL).
1263  // Curl seems to need the just scheme, hostname and a path;
1264  // the rest was already passed as curl options (in attachTo).
1265  Url curlUrl( clearQueryString(url) );
1266 
1267  //
1268  // See also Bug #154197 and ftp url definition in RFC 1738:
1269  // The url "ftp://user@host/foo/bar/file" contains a path,
1270  // that is relative to the user's home.
1271  // The url "ftp://user@host//foo/bar/file" (or also with
1272  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1273  // contains an absolute path.
1274  //
1275  _lastRedirect.clear();
1276  std::string urlBuffer( curlUrl.asString());
1277  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1278  urlBuffer.c_str() );
1279  if ( ret != 0 ) {
1281  }
1282 
1283  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
1284  if ( ret != 0 ) {
1286  }
1287 
1288  // Set callback and perform.
1289  internal::ProgressData progressData(_curl, _settings.timeout(), url, srcFile.downloadSize(), &report);
1290  if (!(options & OPTION_NO_REPORT_START))
1291  report->start(url, dest);
1292  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
1293  WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1294  }
1295 
1296  ret = curl_easy_perform( _curl );
1297 #if CURLVERSION_AT_LEAST(7,19,4)
1298  // bnc#692260: If the client sends a request with an If-Modified-Since header
1299  // with a future date for the server, the server may respond 200 sending a
1300  // zero size file.
1301  // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
1302  if ( ftell(file) == 0 && ret == 0 )
1303  {
1304  long httpReturnCode = 33;
1305  if ( curl_easy_getinfo( _curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
1306  {
1307  long conditionUnmet = 33;
1308  if ( curl_easy_getinfo( _curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
1309  {
1310  WAR << "TIMECONDITION unmet - retry without." << endl;
1311  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1312  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1313  ret = curl_easy_perform( _curl );
1314  }
1315  }
1316  }
1317 #endif
1318 
1319  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
1320  WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1321  }
1322 
1323  if ( ret != 0 )
1324  {
1325  ERR << "curl error: " << ret << ": " << _curlError
1326  << ", temp file size " << ftell(file)
1327  << " bytes." << endl;
1328 
1329  // the timeout is determined by the progress data object
1330  // which holds whether the timeout was reached or not,
1331  // otherwise it would be a user cancel
1332  try {
1333 
1334  if ( progressData.fileSizeExceeded() )
1335  ZYPP_THROW(MediaFileSizeExceededException(url, progressData.expectedFileSize()));
1336 
1337  evaluateCurlCode( srcFile.filename(), ret, progressData.timeoutReached() );
1338  }
1339  catch ( const MediaException &e ) {
1340  // some error, we are not sure about file existence, rethrw
1341  ZYPP_RETHROW(e);
1342  }
1343  }
1344 
1345 #if DETECT_DIR_INDEX
1346  if (!ret && detectDirIndex())
1347  {
1349  }
1350 #endif // DETECT_DIR_INDEX
1351 }
1352 
1354 
1355 void MediaCurl::getDir( const Pathname & dirname, bool recurse_r ) const
1356 {
1357  filesystem::DirContent content;
1358  getDirInfo( content, dirname, /*dots*/false );
1359 
1360  for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
1361  Pathname filename = dirname + it->name;
1362  int res = 0;
1363 
1364  switch ( it->type ) {
1365  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
1366  case filesystem::FT_FILE:
1367  getFile( OnMediaLocation( filename ) );
1368  break;
1369  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
1370  if ( recurse_r ) {
1371  getDir( filename, recurse_r );
1372  } else {
1373  res = assert_dir( localPath( filename ) );
1374  if ( res ) {
1375  WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
1376  }
1377  }
1378  break;
1379  default:
1380  // don't provide devices, sockets, etc.
1381  break;
1382  }
1383  }
1384 }
1385 
1387 
1388 void MediaCurl::getDirInfo( std::list<std::string> & retlist,
1389  const Pathname & dirname, bool dots ) const
1390 {
1391  getDirectoryYast( retlist, dirname, dots );
1392 }
1393 
1395 
1397  const Pathname & dirname, bool dots ) const
1398 {
1399  getDirectoryYast( retlist, dirname, dots );
1400 }
1401 
1403 //
1404 int MediaCurl::aliveCallback( void *clientp, double /*dltotal*/, double dlnow, double /*ultotal*/, double /*ulnow*/ )
1405 {
1406  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1407  if( pdata )
1408  {
1409  // Do not propagate dltotal in alive callbacks. MultiCurl uses this to
1410  // prevent a percentage raise while downloading a metalink file. Download
1411  // activity however is indicated by propagating the download rate (via dlnow).
1412  pdata->updateStats( 0.0, dlnow );
1413  return pdata->reportProgress();
1414  }
1415  return 0;
1416 }
1417 
1418 int MediaCurl::progressCallback( void *clientp, double dltotal, double dlnow, double ultotal, double ulnow )
1419 {
1420  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1421  if( pdata )
1422  {
1423  // work around curl bug that gives us old data
1424  long httpReturnCode = 0;
1425  if ( curl_easy_getinfo( pdata->curl(), CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0 )
1426  return aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
1427 
1428  pdata->updateStats( dltotal, dlnow );
1429  return pdata->reportProgress();
1430  }
1431  return 0;
1432 }
1433 
1435 {
1436  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>(clientp);
1437  return pdata ? pdata->curl() : 0;
1438 }
1439 
1441 
1442 std::string MediaCurl::getAuthHint() const
1443 {
1444  long auth_info = CURLAUTH_NONE;
1445 
1446  CURLcode infoRet =
1447  curl_easy_getinfo(_curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1448 
1449  if(infoRet == CURLE_OK)
1450  {
1451  return CurlAuthData::auth_type_long2str(auth_info);
1452  }
1453 
1454  return "";
1455 }
1456 
1461 void MediaCurl::resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
1462 {
1463  internal::ProgressData *data = reinterpret_cast<internal::ProgressData *>(clientp);
1464  if ( data ) {
1465  data->expectedFileSize( expectedFileSize );
1466  }
1467 }
1468 
1470 
1471 bool MediaCurl::authenticate(const std::string & availAuthTypes, bool firstTry) const
1472 {
1474  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
1475  CurlAuthData_Ptr credentials;
1476 
1477  // get stored credentials
1478  AuthData_Ptr cmcred = cm.getCred(_url);
1479 
1480  if (cmcred && firstTry)
1481  {
1482  credentials.reset(new CurlAuthData(*cmcred));
1483  DBG << "got stored credentials:" << endl << *credentials << endl;
1484  }
1485  // if not found, ask user
1486  else
1487  {
1488 
1489  CurlAuthData_Ptr curlcred;
1490  curlcred.reset(new CurlAuthData());
1492 
1493  // preset the username if present in current url
1494  if (!_url.getUsername().empty() && firstTry)
1495  curlcred->setUsername(_url.getUsername());
1496  // if CM has found some credentials, preset the username from there
1497  else if (cmcred)
1498  curlcred->setUsername(cmcred->username());
1499 
1500  // indicate we have no good credentials from CM
1501  cmcred.reset();
1502 
1503  std::string prompt_msg = str::Format(_("Authentication required for '%s'")) % _url.asString();
1504 
1505  // set available authentication types from the exception
1506  // might be needed in prompt
1507  curlcred->setAuthType(availAuthTypes);
1508 
1509  // ask user
1510  if (auth_report->prompt(_url, prompt_msg, *curlcred))
1511  {
1512  DBG << "callback answer: retry" << endl
1513  << "CurlAuthData: " << *curlcred << endl;
1514 
1515  if (curlcred->valid())
1516  {
1517  credentials = curlcred;
1518  // if (credentials->username() != _url.getUsername())
1519  // _url.setUsername(credentials->username());
1527  }
1528  }
1529  else
1530  {
1531  DBG << "callback answer: cancel" << endl;
1532  }
1533  }
1534 
1535  // set username and password
1536  if (credentials)
1537  {
1538  // HACK, why is this const?
1539  const_cast<MediaCurl*>(this)->_settings.setUsername(credentials->username());
1540  const_cast<MediaCurl*>(this)->_settings.setPassword(credentials->password());
1541 
1542  // set username and password
1543  CURLcode ret = curl_easy_setopt(_curl, CURLOPT_USERPWD, _settings.userPassword().c_str());
1545 
1546  // set available authentication types from the exception
1547  if (credentials->authType() == CURLAUTH_NONE)
1548  credentials->setAuthType(availAuthTypes);
1549 
1550  // set auth type (seems this must be set _after_ setting the userpwd)
1551  if (credentials->authType() != CURLAUTH_NONE)
1552  {
1553  // FIXME: only overwrite if not empty?
1554  const_cast<MediaCurl*>(this)->_settings.setAuthType(credentials->authTypeAsString());
1555  ret = curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, credentials->authType());
1557  }
1558 
1559  if (!cmcred)
1560  {
1561  credentials->setUrl(_url);
1562  cm.addCred(*credentials);
1563  cm.save();
1564  }
1565 
1566  return true;
1567  }
1568 
1569  return false;
1570 }
1571 
1572 //need a out of line definiton, otherwise vtable is emitted for every translation unit
1574 
1575  } // namespace media
1576 } // namespace zypp
1577 //
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:533
long timeout() const
transfer timeout
void globalInitCurlOnce()
Definition: curlhelper.cc:59
Url getFileUrl(const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
virtual bool checkAttachPoint(const Pathname &apoint) const override
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
Definition: MediaCurl.cc:703
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:319
MediaCurlExceptionMayRetryInternaly(const Url &url_r, const std::string &err_r, const std::string &msg_r)
Definition: MediaCurl.cc:300
std::string targetDistribution() const
This is register.target attribute of the installed base product.
Definition: Target.cc:102
#define SET_OPTION_OFFT(opt, val)
Definition: MediaCurl.cc:330
#define MIL
Definition: Logger.h:96
const Pathname & clientCertificatePath() const
SSL client certificate file.
bool progress(int value_r, const Url &file_r, double dbps_avg_r=-1, double dbps_current_r=-1) override
Download progress.
Definition: MediaCurl.cc:83
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar"
#define _(MSG)
Definition: Gettext.h:37
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition: curlhelper.cc:102
ProgressData()
Ctor no range [0,0](0).
Definition: progressdata.h:174
const Pathname & path() const
Return current Pathname.
Definition: PathInfo.h:246
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
Describes a resource file located on a medium.
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:832
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
void checkProtocol(const Url &url) const
check the url is supported by the curl library
Definition: MediaCurl.cc:382
bool authenticate(const std::string &availAuthTypes, bool firstTry) const
Definition: MediaCurl.cc:1471
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl.h:31
void setPassword(const std::string &val_r)
sets the auth password
std::chrono::steady_clock::time_point TimePoint
Definition: MediaCurl.cc:51
ByteCount _expectedFileSize
Definition: MediaCurl.cc:167
virtual bool checkAttachPoint(const Pathname &apoint) const
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
int log_curl(CURL *, curl_infotype info, char *ptr, size_t len, void *max_lvl)
Definition: curlhelper.cc:68
const char * distributionFlavorHeader()
Definition: MediaCurl.cc:265
Store and operate with byte count.
Definition: ByteCount.h:30
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
to not add a IFMODSINCE header if target exists
Definition: MediaCurl.h:43
Holds transfer setting.
void save()
Saves any unsaved credentials added via addUserCred() or addGlobalCred() methods. ...
const std::string & authType() const
get the allowed authentication types
bool verifyHostEnabled() const
Whether to verify host for ssl.
const std::string & proxyUsername() const
proxy auth username
static int progressCallback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
Callback reporting download progress.
Definition: MediaCurl.cc:1418
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:173
int reportProgress() const
Definition: MediaCurl.cc:240
Url clearQueryString(const Url &url)
Definition: curlhelper.cc:332
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
void start(const Url &file_r, Pathname localfile_r) override
Definition: MediaCurl.cc:71
const char * anonymousIdHeader()
Definition: MediaCurl.cc:251
const char * c_str() const
String representation.
Definition: Pathname.h:110
void addHistory(const std::string &msg_r)
Add some message text to the history.
Definition: Exception.cc:140
bool isUseableAttachPoint(const Pathname &path, bool mtab=true) const
Ask media manager, if the specified path is already used as attach point or if there are another atta...
void setUsername(const std::string &val_r)
sets the auth username
Action problem(const Url &file_r, Error error_r, const std::string &description_r) override
Definition: MediaCurl.cc:102
virtual void getFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename) const override
Definition: MediaCurl.cc:745
static int aliveCallback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
Callback sending just an alive trigger to the UI, without stats (e.g.
Definition: MediaCurl.cc:1404
time_t _timeNow
Now.
Definition: MediaCurl.cc:173
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition: ManagedFile.h:27
void setConnectTimeout(long t)
set the connect timeout
virtual void setupEasy()
initializes the curl easy handle with the data from the url
Definition: MediaCurl.cc:407
bool timeoutReached() const
Definition: MediaCurl.cc:149
Convenient building of std::string with boost::format.
Definition: String.h:252
Structure holding values of curlrc options.
Definition: curlconfig.h:26
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:36
Edition * _value
Definition: SysContent.cc:311
AutoDispose<int> calling ::close
Definition: AutoDispose.h:300
std::string _currentCookieFile
Definition: MediaCurl.h:162
const std::string & password() const
auth password
virtual void getDir(const Pathname &dirname, bool recurse_r) const override
Call concrete handler to provide directory content (not recursive!) below attach point.
Definition: MediaCurl.cc:1355
virtual void getFile(const OnMediaLocation &file) const override
Call concrete handler to provide file below attach point.
Definition: MediaCurl.cc:736
#define ERR
Definition: Logger.h:98
const std::string & username() const
auth username
bool fileSizeExceeded() const
Definition: MediaCurl.cc:152
Pathname localPath(const Pathname &pathname) const
Files provided will be available at &#39;localPath(filename)&#39;.
const Headers & headers() const
returns a list of all added headers
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:48
static void setCookieFile(const Pathname &)
Definition: MediaCurl.cc:375
virtual void releaseFrom(const std::string &ejectDev) override
Call concrete handler to release the media.
Definition: MediaCurl.cc:729
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
static void resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
MediaMultiCurl needs to reset the expected filesize in case a metalink file is downloaded otherwise t...
Definition: MediaCurl.cc:1461
const std::string & hint() const
comma separated list of available authentication types
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:440
bool detectDirIndex() const
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: curlconfig.cc:24
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
Definition: PathInfo.cc:1202
double _dnlTotal
Bytes to download or 0 if unknown.
Definition: MediaCurl.cc:175
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:497
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:660
Bottleneck filtering all DownloadProgressReport issued from Media[Muli]Curl.
Definition: MediaCurl.cc:49
Url clearQueryString(const Url &url) const
Definition: MediaCurl.cc:370
void setAuthType(const std::string &val_r)
set the allowed authentication types
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:223
void finish(const Url &file_r, Error error_r, const std::string &reason_r) override
Definition: MediaCurl.cc:108
int unlink(const Pathname &path)
Like &#39;unlink&#39;.
Definition: PathInfo.cc:700
const Url _url
Url to handle.
Definition: MediaHandler.h:113
void setMediaSource(const MediaSourceRef &ref)
Set new media source reference.
const std::string & asString() const
String representation.
Definition: Pathname.h:91
int rename(const Pathname &oldpath, const Pathname &newpath)
Like &#39;rename&#39;.
Definition: PathInfo.cc:742
Just inherits Exception to separate media exceptions.
void evaluateCurlCode(const zypp::Pathname &filename, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition: MediaCurl.cc:840
std::string asUserHistory() const
A single (multiline) string composed of asUserString and historyAsString.
Definition: Exception.cc:91
const ByteCount & downloadSize() const
The size of the resource on the server.
void disconnect()
Use concrete handler to isconnect media.
long connectTimeout() const
connection timeout
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:124
do not send a start ProgressReport
Definition: MediaCurl.h:45
#define WAR
Definition: Logger.h:97
double _dnlLast
Bytes downloaded at period start.
Definition: MediaCurl.cc:176
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:518
void getDirectoryYast(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const
Retrieve and if available scan dirname/directory.yast.
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: curlhelper.cc:142
void setTimeout(long t)
set the transfer timeout
Receiver * _oldRec
std::string proxyuserpwd
Definition: curlconfig.h:49
bool isValid() const
Verifies the Url.
Definition: Url.cc:489
virtual bool doGetDoesFileExist(const Pathname &filename) const
Definition: MediaCurl.cc:988
const Pathname & clientKeyPath() const
SSL client key file.
void expectedFileSize(ByteCount newval_r)
Definition: MediaCurl.cc:158
shared_ptr< CurlAuthData > CurlAuthData_Ptr
Definition: curlauthdata.h:102
const Pathname & filename() const
The path to the resource on the medium.
virtual void attachTo(bool next=false) override
Call concrete handler to attach the media.
Definition: MediaCurl.cc:668
std::string numstring(char n, int w=0)
Definition: String.h:289
Common baseclass for MediaCurl and MediaNetwork.
virtual bool getDoesFileExist(const Pathname &filename) const override
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly, or user cancels the operation.
Definition: MediaCurl.cc:809
const char * agentString()
Definition: MediaCurl.cc:279
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:180
void doGetFileCopyFile(const OnMediaLocation &srcFile, const Pathname &dest, FILE *file, callback::SendReport< DownloadProgressReport > &report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1248
Media source internally used by MediaManager and MediaHandler.
Definition: MediaSource.h:36
static std::string auth_type_long2str(long auth_type)
Converts a long of ORed CURLAUTH_* identifiers into a string of comma separated list of authenticatio...
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
int _dnlPercent
Percent completed or 0 if _dnlTotal is unknown.
Definition: MediaCurl.cc:179
curl_slist * _customHeaders
Definition: MediaCurl.h:170
bool proxyEnabled() const
proxy is enabled
long ZYPP_MEDIA_CURL_DEBUG()
Long number for setting CURLOPT_DEBUGDATA.
Definition: curlhelper_p.h:35
shared_ptr< AuthData > AuthData_Ptr
Definition: authdata.h:79
int rmdir(const Pathname &path)
Like &#39;rmdir&#39;.
Definition: PathInfo.cc:366
#define SET_OPTION(opt, val)
Definition: MediaCurl.cc:323
Pathname absolutename() const
Return this path, adding a leading &#39;/&#39; if relative.
Definition: Pathname.h:139
Base class for Exception.
Definition: Exception.h:145
Pathname attachPoint() const
Return the currently used attach point.
std::string _lastRedirect
to log/report redirections
Definition: MediaCurl.h:165
Url url() const
Url used.
Definition: MediaHandler.h:503
double _drateLast
Download rate in last period.
Definition: MediaCurl.cc:182
std::string curlUnEscape(std::string text_r)
Definition: curlhelper.cc:325
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:588
time_t _timeStart
Start total stats.
Definition: MediaCurl.cc:170
virtual void getDirInfo(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const override
Call concrete handler to provide a content list of directory on media via retlist.
Definition: MediaCurl.cc:1388
virtual void disconnectFrom() override
Definition: MediaCurl.cc:710
static CURL * progressCallback_getcurl(void *clientp)
Definition: MediaCurl.cc:1434
Typesafe passing of user data via callbacks.
Definition: UserData.h:38
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
double _dnlNow
Bytes downloaded now.
Definition: MediaCurl.cc:177
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3"
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:220
void report(const UserData &userData_r=UserData()) override
The most generic way of sending/receiving data.
Definition: MediaCurl.cc:67
AutoDispose<FILE*> calling ::fclose
Definition: AutoDispose.h:311
void updateStats(double dltotal=0.0, double dlnow=0.0)
Definition: MediaCurl.cc:197
static Pathname _cookieFile
Definition: MediaCurl.h:163
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: curlhelper.cc:290
double _drateTotal
Download rate so far.
Definition: MediaCurl.cc:181
std::string anonymousUniqueId() const
anonymous unique id
Definition: Target.cc:132
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:789
std::string userPassword() const
returns the user and password as a user:pass string
std::string getAuthHint() const
Return a comma separated list of available authentication methods supported by server.
Definition: MediaCurl.cc:1442
time_t _timeRcv
Start of no-data timeout.
Definition: MediaCurl.cc:172
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:25
Pathname createAttachPoint() const
Try to create a default / temporary attach point.
virtual void doGetFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename, callback::SendReport< DownloadProgressReport > &_report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1138
void addCred(const AuthData &cred)
Add new credentials with user callbacks.
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
Curl HTTP authentication data.
Definition: curlauthdata.h:22
std::string distributionFlavor() const
This is flavor attribute of the installed base product but does not require the target to be loaded a...
Definition: Target.cc:127
OptionalDownloadProgressReport(bool isOptional=false)
Definition: MediaCurl.cc:53
char _curlError[CURL_ERROR_SIZE]
Definition: MediaCurl.h:169
ByteCount expectedFileSize() const
Definition: MediaCurl.cc:155
#define CONNECT_TIMEOUT
Definition: curlhelper_p.h:21
const std::string & proxy() const
proxy host
MediaCurlException(const Url &url_r, const std::string &err_r, const std::string &msg_r)
bool optional() const
Whether this is an optional resource.
time_t _timeLast
Start last period(~1sec)
Definition: MediaCurl.cc:171
const char * c_str() const
Definition: IdStringType.h:105
TrueBool _guard
Definition: TargetImpl.cc:1563
bool userMayRWX() const
Definition: PathInfo.h:353
const std::string & userAgentString() const
user agent string
Url manipulation class.
Definition: Url.h:91
bool headRequestsAllowed() const
whether HEAD requests are allowed
#define DBG
Definition: Logger.h:95
Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy E.g. ...
Definition: MediaCurl.cc:297
zypp::callback::SendReport< zypp::media::DownloadProgressReport > * report
Definition: MediaCurl.cc:168
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:572