17 #include <zypp/base/Logger.h> 18 #include <zypp/ExternalProgram.h> 19 #include <zypp/base/String.h> 20 #include <zypp/base/Gettext.h> 21 #include <zypp-core/parser/Sysconfig> 22 #include <zypp/base/Gettext.h> 25 #include <zypp-curl/ProxyInfo> 26 #include <zypp-curl/auth/CurlAuthData> 27 #include <zypp-media/auth/CredentialManager> 28 #include <zypp-curl/CurlConfig> 35 #include <sys/types.h> 37 #include <sys/mount.h> 51 using TimePoint = std::chrono::steady_clock::time_point;
54 :
_oldRec { Distributor::instance().getReceiver() }
55 , _isOptional { isOptional }
59 {
if (
_oldRec ) Distributor::instance().setReceiver( *
_oldRec );
else Distributor::instance().noReceiver(); }
71 void start(
const Url & file_r, Pathname localfile_r )
override 77 _startLocalfile = std::move(localfile_r);
80 _oldRec->start( file_r, localfile_r );
83 bool progress(
int value_r,
const Url & file_r,
double dbps_avg_r = -1,
double dbps_current_r = -1 )
override 87 if ( not ( value_r || dbps_avg_r || dbps_current_r ) )
93 static constexpr std::chrono::milliseconds maxfequency { 100 };
94 TimePoint now { TimePoint::clock::now() };
95 TimePoint::duration elapsed { now - _lastProgressSent };
96 if ( elapsed < maxfequency )
98 _lastProgressSent = now;
99 return _oldRec->progress( value_r, file_r, dbps_avg_r, dbps_current_r );
104 if ( not
_oldRec || notStarted() )
return ABORT;
105 return _oldRec->problem( file_r, error_r, description_r );
108 void finish(
const Url & file_r,
Error error_r,
const std::string & reason_r )
override 110 if ( not
_oldRec || notStarted() )
return;
111 _oldRec->finish( file_r, error_r, reason_r );
117 {
return _isOptional; }
123 _oldRec->start( std::move(_startFile), std::move(_startLocalfile) );
142 void updateStats(
double dltotal = 0.0,
double dlnow = 0.0 );
144 int reportProgress()
const;
150 {
return _timeoutReached; }
153 {
return _fileSizeExceeded; }
156 {
return _expectedFileSize; }
159 { _expectedFileSize = newval_r; }
170 time_t _timeStart = 0;
171 time_t _timeLast = 0;
175 double _dnlTotal = 0.0;
176 double _dnlLast = 0.0;
177 double _dnlNow = 0.0;
181 double _drateTotal= 0.0;
182 double _drateLast = 0.0;
190 , _timeout( timeout )
191 , _timeoutReached( false )
192 , _fileSizeExceeded ( false )
193 , _expectedFileSize( expectedFileSize_r )
205 if ( dlnow && dlnow !=
_dnlNow )
257 static const std::string
_value(
259 "X-ZYpp-AnonymousId: %s",
271 static const std::string
_value(
273 "X-ZYpp-DistributionFlavor: %s",
285 static const std::string
_value(
287 "ZYpp " LIBZYPP_VERSION_STRING
" (curl %s) %s" 288 , curl_version_info(CURLVERSION_NOW)->version
301 const std::string & err_r,
302 const std::string & msg_r )
318 Pathname MediaCurl::_cookieFile =
"/var/lib/YaST2/cookies";
323 #define SET_OPTION(opt,val) do { \ 324 ret = curl_easy_setopt ( _curl, opt, val ); \ 326 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError)); \ 330 #define SET_OPTION_OFFT(opt,val) SET_OPTION(opt,(curl_off_t)val) 331 #define SET_OPTION_LONG(opt,val) SET_OPTION(opt,(long)val) 332 #define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val) 334 MediaCurl::MediaCurl(
const Url & url_r,
335 const Pathname & attach_point_hint_r )
345 MIL <<
"MediaCurl::MediaCurl(" << url_r <<
", " << attach_point_hint_r <<
")" << endl;
353 char *atemp = ::strdup( apath.
asString().c_str());
356 atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
358 WAR <<
"attach point " << ainfo.
path()
359 <<
" is not useable for " << url_r.
getScheme() << endl;
362 else if( atest != NULL)
384 curl_version_info_data *curl_info = NULL;
385 curl_info = curl_version_info(CURLVERSION_NOW);
387 if (curl_info->protocols)
389 const char *
const *proto;
392 for(proto=curl_info->protocols; !found && *proto; ++proto)
394 if( scheme == std::string((
const char *)*proto))
399 std::string msg(
"Unsupported protocol '");
413 curl_easy_setopt(
_curl, CURLOPT_VERBOSE, 1L);
421 CURLcode ret = curl_easy_setopt(
_curl, CURLOPT_ERRORBUFFER,
_curlError );
466 case 4:
SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
break;
467 case 6:
SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6);
break;
490 #if CURLVERSION_AT_LEAST(7,19,4) 493 SET_OPTION( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS );
495 SET_OPTION( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
513 #ifdef CURLSSLOPT_ALLOW_BEAST 515 ret = curl_easy_setopt(
_curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
524 SET_OPTION(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
542 if ( cred && cred->valid() ) {
560 if (use_auth.empty())
561 use_auth =
"digest,basic";
563 if( auth != CURLAUTH_NONE)
565 DBG <<
"Enabling HTTP authentication methods: " << use_auth
566 <<
" (CURLOPT_HTTPAUTH=" << auth <<
")" << std::endl;
575 SET_OPTION(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
585 if ( proxyuserpwd.empty() )
590 DBG <<
"Proxy: ~/.curlrc does not contain the proxy-user option" << endl;
594 DBG <<
"Proxy: using proxy-user from ~/.curlrc" << endl;
602 if ( ! proxyuserpwd.empty() )
607 #if CURLVERSION_AT_LEAST(7,19,4) 612 DBG <<
"Proxy: explicitly NOPROXY" << endl;
618 DBG <<
"Proxy: not explicitly set" << endl;
619 DBG <<
"Proxy: libcurl may look into the environment" << endl;
630 #if CURLVERSION_AT_LEAST(7,15,5) 643 MIL <<
"No cookies requested" << endl;
648 #if CURLVERSION_AT_LEAST(7,18,0) 653 for (
const auto &header : vol_settings.
headers() )
683 _curl = curl_easy_init();
721 try { curl_easy_cleanup(
_curl ); }
748 const auto &filename = srcFile.
filename();
757 unsigned internalTry = 0;
758 static constexpr
unsigned maxInternalTry = 3;
783 if ( internalTry < maxInternalTry ) {
842 bool timeout_reached)
const 847 if (filename.
empty())
856 case CURLE_UNSUPPORTED_PROTOCOL:
857 err =
" Unsupported protocol";
860 err +=
" or redirect (";
865 case CURLE_URL_MALFORMAT:
866 case CURLE_URL_MALFORMAT_USER:
869 case CURLE_LOGIN_DENIED:
873 case CURLE_HTTP_RETURNED_ERROR:
875 long httpReturnCode = 0;
876 CURLcode infoRet = curl_easy_getinfo(
_curl,
877 CURLINFO_RESPONSE_CODE,
879 if ( infoRet == CURLE_OK )
881 std::string msg =
"HTTP response: " +
str::numstring( httpReturnCode );
882 switch ( httpReturnCode )
888 DBG << msg <<
" Login failed (URL: " <<
url.
asString() <<
")" << std::endl;
889 DBG <<
"MediaUnauthorizedException auth hint: '" << auth_hint <<
"'" << std::endl;
904 if (
url.
getHost().find(
".suse.com") != std::string::npos )
905 msg403 =
_(
"Visit the SUSE Customer Center to check whether your registration is valid and has not expired.");
906 else if (
url.
asString().find(
"novell.com") != std::string::npos)
907 msg403 =
_(
"Visit the Novell Customer Center to check whether your registration is valid and has not expired.");
920 std::string msg =
"Unable to retrieve HTTP response:";
926 case CURLE_FTP_COULDNT_RETR_FILE:
927 #if CURLVERSION_AT_LEAST(7,16,0) 928 case CURLE_REMOTE_FILE_NOT_FOUND:
930 case CURLE_FTP_ACCESS_DENIED:
931 case CURLE_TFTP_NOTFOUND:
932 err =
"File not found";
935 case CURLE_BAD_PASSWORD_ENTERED:
936 case CURLE_FTP_USER_PASSWORD_INCORRECT:
937 err =
"Login failed";
939 case CURLE_COULDNT_RESOLVE_PROXY:
940 case CURLE_COULDNT_RESOLVE_HOST:
941 case CURLE_COULDNT_CONNECT:
942 case CURLE_FTP_CANT_GET_HOST:
943 err =
"Connection failed";
945 case CURLE_WRITE_ERROR:
948 case CURLE_PARTIAL_FILE:
949 case CURLE_OPERATION_TIMEDOUT:
950 timeout_reached =
true;
952 case CURLE_ABORTED_BY_CALLBACK:
953 if( timeout_reached )
955 err =
"Timeout reached";
966 case CURLE_HTTP2_STREAM:
1016 std::string urlBuffer( curlUrl.
asString());
1017 CURLcode ret = curl_easy_setopt(
_curl, CURLOPT_URL,
1018 urlBuffer.c_str() );
1030 struct TempSetHeadRequest
1032 TempSetHeadRequest( CURL * curl_r,
bool doHttpHeadRequest_r )
1034 , _doHttpHeadRequest { doHttpHeadRequest_r }
1036 if ( _doHttpHeadRequest ) {
1037 curl_easy_setopt(
_curl, CURLOPT_NOBODY, 1L );
1039 curl_easy_setopt(
_curl, CURLOPT_RANGE,
"0-1" );
1042 ~TempSetHeadRequest() {
1043 if ( _doHttpHeadRequest ) {
1044 curl_easy_setopt(
_curl, CURLOPT_NOBODY, 0L);
1050 curl_easy_setopt(
_curl, CURLOPT_HTTPGET, 1L );
1052 curl_easy_setopt(
_curl, CURLOPT_RANGE, NULL );
1057 bool _doHttpHeadRequest;
1061 AutoFILE file { ::fopen(
"/dev/null",
"w" ) };
1063 ERR <<
"fopen failed for /dev/null" << endl;
1067 ret = curl_easy_setopt(
_curl, CURLOPT_WRITEDATA, (*file) );
1072 CURLcode ok = curl_easy_perform(
_curl );
1073 MIL <<
"perform code: " << ok <<
" [ " << curl_easy_strerror(ok) <<
" ]" << endl;
1090 return ( ok == CURLE_OK );
1096 #if DETECT_DIR_INDEX 1109 bool not_a_file =
false;
1111 CURLcode ret = curl_easy_getinfo(
_curl,
1112 CURLINFO_EFFECTIVE_URL,
1114 if ( ret == CURLE_OK && ptr != NULL)
1119 std::string path( eurl.getPathName());
1120 if( !path.empty() && path !=
"/" && *path.rbegin() ==
'/')
1122 DBG <<
"Effective url (" 1124 <<
") seems to provide the index of a directory" 1143 DBG <<
"assert_dir " << dest.
dirname() <<
" failed" << endl;
1153 ERR <<
"out of memory for temp file name" << endl;
1157 AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1160 ERR <<
"mkstemp failed for file '" << destNew <<
"'" << endl;
1165 file = ::fdopen( tmp_fd,
"we" );
1168 ERR <<
"fopen failed for file '" << destNew <<
"'" << endl;
1171 tmp_fd.resetDispose();
1174 DBG <<
"dest: " << dest << endl;
1175 DBG <<
"temp: " << destNew << endl;
1180 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1181 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, (
long)
PathInfo(target).mtime());
1185 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1186 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
1194 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1195 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
1199 long httpReturnCode = 0;
1200 CURLcode infoRet = curl_easy_getinfo(
_curl,
1201 CURLINFO_RESPONSE_CODE,
1203 bool modified =
true;
1204 if (infoRet == CURLE_OK)
1207 if ( httpReturnCode == 304
1210 DBG <<
" Not modified.";
1217 WAR <<
"Could not get the response code." << endl;
1220 if (modified || infoRet != CURLE_OK)
1225 ERR <<
"Failed to chmod file " << destNew << endl;
1229 if ( ::fclose( file ) )
1231 ERR <<
"Fclose failed for file '" << destNew <<
"'" << endl;
1236 if (
rename( destNew, dest ) != 0 ) {
1237 ERR <<
"Rename failed" << endl;
1240 destNew.resetDispose();
1276 std::string urlBuffer( curlUrl.
asString());
1277 CURLcode ret = curl_easy_setopt(
_curl, CURLOPT_URL,
1278 urlBuffer.c_str() );
1283 ret = curl_easy_setopt(
_curl, CURLOPT_WRITEDATA, file );
1291 report->start(
url, dest);
1292 if ( curl_easy_setopt(
_curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
1293 WAR <<
"Can't set CURLOPT_PROGRESSDATA: " <<
_curlError << endl;;
1296 ret = curl_easy_perform(
_curl );
1297 #if CURLVERSION_AT_LEAST(7,19,4) 1302 if ( ftell(file) == 0 && ret == 0 )
1304 long httpReturnCode = 33;
1305 if ( curl_easy_getinfo(
_curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
1307 long conditionUnmet = 33;
1308 if ( curl_easy_getinfo(
_curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
1310 WAR <<
"TIMECONDITION unmet - retry without." << endl;
1311 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1312 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
1313 ret = curl_easy_perform(
_curl );
1319 if ( curl_easy_setopt(
_curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
1320 WAR <<
"Can't unset CURLOPT_PROGRESSDATA: " <<
_curlError << endl;;
1326 <<
", temp file size " << ftell(file)
1327 <<
" bytes." << endl;
1334 if ( progressData.fileSizeExceeded() )
1345 #if DETECT_DIR_INDEX 1350 #endif // DETECT_DIR_INDEX 1360 for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
1361 Pathname filename = dirname + it->name;
1364 switch ( it->type ) {
1371 getDir( filename, recurse_r );
1375 WAR <<
"Ignore error (" << res <<
") on creating local directory '" <<
localPath( filename ) <<
"'" << endl;
1389 const Pathname & dirname,
bool dots )
const 1397 const Pathname & dirname,
bool dots )
const 1424 long httpReturnCode = 0;
1425 if ( curl_easy_getinfo( pdata->
curl(), CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0 )
1426 return aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
1437 return pdata ? pdata->
curl() : 0;
1444 long auth_info = CURLAUTH_NONE;
1447 curl_easy_getinfo(
_curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1449 if(infoRet == CURLE_OK)
1480 if (cmcred && firstTry)
1483 DBG <<
"got stored credentials:" << endl << *credentials << endl;
1498 curlcred->setUsername(cmcred->username());
1507 curlcred->setAuthType(availAuthTypes);
1510 if (auth_report->prompt(
_url, prompt_msg, *curlcred))
1512 DBG <<
"callback answer: retry" << endl
1513 <<
"CurlAuthData: " << *curlcred << endl;
1515 if (curlcred->valid())
1517 credentials = curlcred;
1531 DBG <<
"callback answer: cancel" << endl;
1547 if (credentials->authType() == CURLAUTH_NONE)
1548 credentials->setAuthType(availAuthTypes);
1551 if (credentials->authType() != CURLAUTH_NONE)
1555 ret = curl_easy_setopt(
_curl, CURLOPT_HTTPAUTH, credentials->authType());
1561 credentials->setUrl(
_url);
std::string getScheme() const
Returns the scheme name of the URL.
void globalInitCurlOnce()
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
std::string targetDistribution() const
This is register.target attribute of the installed base product.
bool progress(int value_r, const Url &file_r, double dbps_avg_r=-1, double dbps_current_r=-1) override
Download progress.
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
ProgressData()
Ctor no range [0,0](0).
const Pathname & path() const
Return current Pathname.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
static ZConfig & instance()
Singleton ctor.
std::chrono::steady_clock::time_point TimePoint
ByteCount _expectedFileSize
int log_curl(CURL *, curl_infotype info, char *ptr, size_t len, void *max_lvl)
const char * distributionFlavorHeader()
Store and operate with byte count.
void reportend() override
~OptionalDownloadProgressReport()
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
int reportProgress() const
Url clearQueryString(const Url &url)
void start(const Url &file_r, Pathname localfile_r) override
const char * anonymousIdHeader()
const char * c_str() const
String representation.
void addHistory(const std::string &msg_r)
Add some message text to the history.
Action problem(const Url &file_r, Error error_r, const std::string &description_r) override
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
bool timeoutReached() const
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
AutoDispose<int> calling ::close
bool fileSizeExceeded() const
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
void reportbegin() override
bool empty() const
Test for an empty path.
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
double _dnlTotal
Bytes to download or 0 if unknown.
std::string asString() const
Returns a default string representation of the Url object.
std::string getQueryParam(const std::string ¶m, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Bottleneck filtering all DownloadProgressReport issued from Media[Muli]Curl.
std::string trim(const std::string &s, const Trim trim_r)
void finish(const Url &file_r, Error error_r, const std::string &reason_r) override
int unlink(const Pathname &path)
Like 'unlink'.
const std::string & asString() const
String representation.
int rename(const Pathname &oldpath, const Pathname &newpath)
Like 'rename'.
std::string asUserHistory() const
A single (multiline) string composed of asUserString and historyAsString.
Pathname dirname() const
Return all but the last component od this path.
double _dnlLast
Bytes downloaded at period start.
std::list< DirEntry > DirContent
Returned by readdir.
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
bool isValid() const
Verifies the Url.
void expectedFileSize(ByteCount newval_r)
std::string numstring(char n, int w=0)
const char * agentString()
void resetDispose()
Set no dispose function.
TimePoint _lastProgressSent
int _dnlPercent
Percent completed or 0 if _dnlTotal is unknown.
long ZYPP_MEDIA_CURL_DEBUG()
Long number for setting CURLOPT_DEBUGDATA.
int rmdir(const Pathname &path)
Like 'rmdir'.
Pathname absolutename() const
Return this path, adding a leading '/' if relative.
Base class for Exception.
double _drateLast
Download rate in last period.
std::string curlUnEscape(std::string text_r)
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
time_t _timeStart
Start total stats.
Typesafe passing of user data via callbacks.
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
double _dnlNow
Bytes downloaded now.
Wrapper class for ::stat/::lstat.
void report(const UserData &userData_r=UserData()) override
The most generic way of sending/receiving data.
AutoDispose<FILE*> calling ::fclose
void updateStats(double dltotal=0.0, double dlnow=0.0)
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
double _drateTotal
Download rate so far.
std::string anonymousUniqueId() const
anonymous unique id
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
time_t _timeRcv
Start of no-data timeout.
#define EXPLICITLY_NO_PROXY
Easy-to use interface to the ZYPP dependency resolver.
std::string distributionFlavor() const
This is flavor attribute of the installed base product but does not require the target to be loaded a...
OptionalDownloadProgressReport(bool isOptional=false)
ByteCount expectedFileSize() const
time_t _timeLast
Start last period(~1sec)
const char * c_str() const
zypp::callback::SendReport< zypp::media::DownloadProgressReport > * report
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.