libzypp 17.36.2
curlhelper.cc
Go to the documentation of this file.
1/*---------------------------------------------------------------------\
2| ____ _ __ __ ___ |
3| |__ / \ / / . \ . \ |
4| / / \ V /| _/ _/ |
5| / /__ | | | | | | |
6| /_____||_| |_| |_| |
7| |
8\---------------------------------------------------------------------*/
13
14#include <zypp-core/Globals.h>
15
17#include <zypp-core/Pathname.h>
21#include <zypp-curl/ProxyInfo>
22#include <zypp-curl/auth/CurlAuthData>
23#include <zypp-media/MediaException>
24#include <string>
25#include <glib.h>
26
27#define TRANSFER_TIMEOUT_MAX 60 * 60
28
29using std::endl;
30using namespace zypp;
31
32namespace zypp
33{
34 namespace env
35 {
36 const long & ZYPP_MEDIA_CURL_DEBUG()
37 {
38 static const long ret = [](){
39 const char * env = getenv("ZYPP_MEDIA_CURL_DEBUG");
40 return env && *env ? str::strtonum<ulong>( env ) : 0;
41 }();
42 return ret;
43 }
44
46 {
47 static int _v = [](){
48 int ret = 0;
49 if ( const char * envp = getenv( "ZYPP_MEDIA_CURL_IPRESOLVE" ) ) {
50 WAR << "env set: $ZYPP_MEDIA_CURL_IPRESOLVE='" << envp << "'" << std::endl;
51 if ( strcmp( envp, "4" ) == 0 ) ret = 4;
52 else if ( strcmp( envp, "6" ) == 0 ) ret = 6;
53 }
54 return ret;
55 }();
56 return _v;
57 }
58 } // namespace env
59} // namespace zypp
60
61namespace internal
62{
63
65{
66 // function-level static <=> std::call_once
67 static bool once __attribute__ ((__unused__)) = ( [] {
68 MIL << "global_init libcurl: build version: (" << LIBCURL_VERSION << "), runtime version: (" << curl_version_info(CURLVERSION_NOW)->version << ") " << endl;
69 if ( curl_global_init( CURL_GLOBAL_ALL ) != 0 )
70 WAR << "curl global init failed" << std::endl;
71 } (), true );
72}
73
75{
76 auto curlV = curl_version_info ( CURLVERSION_NOW );
77 return curlV->version_num;
78}
79
80int log_curl( CURL * curl, curl_infotype info, char * ptr, size_t len, void * max_lvl )
81{
82 if ( max_lvl == nullptr )
83 return 0;
84
85 long maxlvl = *(static_cast<long*>(max_lvl));
86 const char * pfx = "";
87 bool isContent = true; // otherwise it's data
88 switch( info )
89 {
90 case CURLINFO_TEXT: if ( maxlvl < 1 ) return 0; pfx = "*"; break;
91 case CURLINFO_HEADER_IN: if ( maxlvl < 2 ) return 0; pfx = "<"; break;
92 case CURLINFO_HEADER_OUT: if ( maxlvl < 2 ) return 0; pfx = ">"; break;
93 case CURLINFO_SSL_DATA_IN: if ( maxlvl < 3 ) return 0; isContent = false; pfx = "<[SSL]"; break;
94 case CURLINFO_SSL_DATA_OUT: if ( maxlvl < 3 ) return 0; isContent = false; pfx = ">[SSL]"; break;
95 case CURLINFO_DATA_IN: if ( maxlvl < 3 ) return 0; isContent = false; pfx = "<[DTA]"; break;
96 case CURLINFO_DATA_OUT: if ( maxlvl < 3 ) return 0; isContent = false; pfx = ">[DTA]"; break;
97
98 default:
99 return 0;
100 }
101
102 // We'd like to keep all log messages within function `log_curl`
103 // because this tag to grep for is known and communicate to users.
104 if ( isContent ) {
105 std::vector<std::string_view> lines; // don't want log from within the lambda
106 strv::split( std::string_view( ptr, len ), "\n", [&lines]( std::string_view line, unsigned, bool last ) {
107 if ( last ) return; // empty word after final \n
108 line = strv::rtrim( line, "\r" );
109 lines.push_back( line );
110 });
111 for ( const auto & line : lines ) {
112 if ( str::hasPrefix( line, "Authorization:" ) ) {
113 std::string_view::size_type pos { line.find( " ", 15 ) }; // Authorization: <type> <credentials>
114 if ( pos == std::string::npos )
115 pos = 15;
116 DBG << curl << " " << pfx << " " << line.substr( 0, pos ) << " <credentials removed>" << endl;
117 }
118 else
119 DBG << curl << " " << pfx << " " << line << endl;
120 }
121 } else {
122 if ( maxlvl < 4 )
123 DBG << curl << " " << pfx << " " << len << " byte" << endl;
124 else
125 hexdumpOn( DBG << curl << " " << pfx << " ", ptr, len );
126 }
127 return 0;
128}
129
131{
132 if ( not curl ) {
133 INT << "Got a NULL curl handle" << endl;
134 return;
135 }
136 if ( env::ZYPP_MEDIA_CURL_DEBUG() > 0 ) {
137 curl_easy_setopt( curl, CURLOPT_VERBOSE, 1L );
138 curl_easy_setopt( curl, CURLOPT_DEBUGFUNCTION, log_curl );
139 curl_easy_setopt( curl, CURLOPT_DEBUGDATA, &env::ZYPP_MEDIA_CURL_DEBUG() );
140 }
141}
142
143size_t log_redirects_curl( char *ptr, size_t size, size_t nmemb, void *userdata)
144{
145 //INT << "got header: " << std::string(ptr, ptr + size*nmemb) << endl;
146
147 char * lstart = ptr, * lend = ptr;
148 size_t pos = 0;
149 size_t max = size * nmemb;
150 while (pos + 1 < max)
151 {
152 // get line
153 for (lstart = lend; *lend != '\n' && pos < max; ++lend, ++pos);
154
155 // look for "Location"
156 if ( strncasecmp( lstart, "Location:", 9 ) == 0 )
157 {
158 std::string line { lstart, *(lend-1)=='\r' ? lend-1 : lend };
159 DBG << "redirecting to " << line << std::endl;
160 if ( userdata ) {
161 *reinterpret_cast<std::string *>( userdata ) = line;
162 }
163 return max;
164 }
165
166 // continue with the next line
167 if (pos + 1 < max)
168 {
169 ++lend;
170 ++pos;
171 }
172 else
173 break;
174 }
175
176 return max;
177}
178
184{
185 {
186 const std::string & param { url.getQueryParam("timeout") };
187 if( ! param.empty() )
188 {
189 long num = str::strtonum<long>(param);
190 if( num >= 0 && num <= TRANSFER_TIMEOUT_MAX )
191 s.setTimeout( num );
192 }
193 }
194 {
195 std::string param { url.getUsername() };
196 if ( ! param.empty() )
197 {
198 s.setUsername( std::move(param) );
199 param = url.getPassword();
200 if ( ! param.empty() )
201 s.setPassword( std::move(param) );
202 }
203 else
204 {
205 // if there is no username, set anonymous auth
206 if ( ( url.getScheme() == "ftp" || url.getScheme() == "tftp" ) && s.username().empty() )
208 }
209 }
210 if ( url.getScheme() == "https" )
211 {
212 s.setVerifyPeerEnabled( false );
213 s.setVerifyHostEnabled( false );
214
215 const std::string & verify { url.getQueryParam("ssl_verify") };
216 if( verify.empty() || verify == "yes" )
217 {
218 s.setVerifyPeerEnabled( true );
219 s.setVerifyHostEnabled( true );
220 }
221 else if ( verify == "no" )
222 {
223 s.setVerifyPeerEnabled( false );
224 s.setVerifyHostEnabled( false );
225 }
226 else
227 {
228 std::vector<std::string> flags;
229 str::split( verify, std::back_inserter(flags), "," );
230 for ( const auto & flag : flags )
231 {
232 if ( flag == "host" )
233 s.setVerifyHostEnabled( true );
234 else if ( flag == "peer" )
235 s.setVerifyPeerEnabled( true );
236 else
237 ZYPP_THROW( media::MediaBadUrlException(url, "Unknown ssl_verify flag "+flag) );
238 }
239 }
240 }
241 {
242 Pathname ca_path { url.getQueryParam("ssl_capath") };
243 if( ! ca_path.empty() )
244 {
245 if( ! PathInfo(ca_path).isDir() || ! ca_path.absolute() )
246 ZYPP_THROW(media::MediaBadUrlException(url, "Invalid ssl_capath path"));
247 else
248 s.setCertificateAuthoritiesPath( std::move(ca_path) );
249 }
250 }
251 {
252 Pathname client_cert { url.getQueryParam("ssl_clientcert") };
253 if( ! client_cert.empty() )
254 {
255 if( ! PathInfo(client_cert).isFile() || ! client_cert.absolute() )
256 ZYPP_THROW(media::MediaBadUrlException(url, "Invalid ssl_clientcert file"));
257 else
258 s.setClientCertificatePath( std::move(client_cert) );
259 }
260 }
261 {
262 Pathname client_key { url.getQueryParam("ssl_clientkey") };
263 if( ! client_key.empty() )
264 {
265 if( ! PathInfo(client_key).isFile() || ! client_key.absolute() )
266 ZYPP_THROW(media::MediaBadUrlException(url, "Invalid ssl_clientkey file"));
267 else
268 s.setClientKeyPath( std::move(client_key) );
269 }
270 }
271 {
272 std::string param { url.getQueryParam( "proxy" ) };
273 if ( ! param.empty() )
274 {
275 if ( param == EXPLICITLY_NO_PROXY ) {
276 // Workaround TransferSettings shortcoming: With an
277 // empty proxy string, code will continue to look for
278 // valid proxy settings. So set proxy to some non-empty
279 // string, to indicate it has been explicitly disabled.
281 s.setProxyEnabled(false);
282 }
283 else {
284 const std::string & proxyport { url.getQueryParam( "proxyport" ) };
285 if ( ! proxyport.empty() ) {
286 param += ":";
287 param += proxyport;
288 }
289 s.setProxy( std::move(param) );
290 s.setProxyEnabled( true );
291 }
292 }
293 }
294 {
295 std::string param { url.getQueryParam( "proxyuser" ) };
296 if ( ! param.empty() )
297 {
298 s.setProxyUsername( std::move(param) );
299 s.setProxyPassword( url.getQueryParam( "proxypass" ) );
300 }
301 }
302 {
303 // HTTP authentication type
304 std::string param { url.getQueryParam("auth") };
305 if ( ! param.empty() && (url.getScheme() == "http" || url.getScheme() == "https") )
306 {
307 try
308 {
309 media::CurlAuthData::auth_type_str2long (param ); // check if we know it
310 }
311 catch ( const media::MediaException & ex_r )
312 {
313 DBG << "Rethrowing as MediaUnauthorizedException.";
315 }
316 s.setAuthType( std::move(param) );
317 }
318 }
319 {
320 // workarounds
321 const std::string & param { url.getQueryParam("head_requests") };
322 if( ! param.empty() && param == "no" )
323 s.setHeadRequestsAllowed( false );
324 }
325}
326
332{
333 media::ProxyInfo proxy_info;
334 if ( proxy_info.useProxyFor( url ) )
335 {
336 // We must extract any 'user:pass' from the proxy url
337 // otherwise they won't make it into curl (.curlrc wins).
338 try {
339 Url u( proxy_info.proxy( url ) );
340 s.setProxy( u.asString( url::ViewOption::WITH_SCHEME + url::ViewOption::WITH_HOST + url::ViewOption::WITH_PORT ) );
341 // don't overwrite explicit auth settings
342 if ( s.proxyUsername().empty() )
343 {
346 }
347 s.setProxyEnabled( true );
348 }
349 catch (...) {} // no proxy if URL is malformed
350 }
351}
352
353void curlEscape( std::string & str_r,
354 const char char_r, const std::string & escaped_r ) {
355 for ( std::string::size_type pos = str_r.find( char_r );
356 pos != std::string::npos; pos = str_r.find( char_r, pos ) ) {
357 str_r.replace( pos, 1, escaped_r );
358 }
359}
360
361std::string curlEscapedPath( std::string path_r ) {
362 curlEscape( path_r, ' ', "%20" );
363 return path_r;
364}
365
366std::string curlUnEscape( const std::string& text_r ) {
367 char * tmp = curl_unescape( text_r.c_str(), 0 );
368 std::string ret( tmp );
369 curl_free( tmp );
370 return ret;
371}
372
374{
375 Url curlUrl (url);
376 curlUrl.setUsername( "" );
377 curlUrl.setPassword( "" );
378 curlUrl.setPathParams( "" );
379 curlUrl.setFragment( "" );
380 curlUrl.delQueryParams( {
381 "cookies",
382 "proxy",
383 "proxyport",
384 "proxyuser",
385 "proxypass",
386 "ssl_capath",
387 "ssl_verify",
388 "ssl_clientcert",
389 "ssl_clientkey",
390 "timeout",
391 "auth",
392 "username",
393 "password",
394 "mediahandler",
395 "credentials",
396 "head_requests",
397 } );
398 return curlUrl;
399}
400
401// bsc#933839: propagate proxy settings passed in the repo URL
402// boo#1127591: propagate ssl settings passed in the repo URL
404{
405 using namespace std::literals::string_literals;
406 for ( const std::string &param : { "proxy"s, "proxyport"s, "proxyuser"s, "proxypass"s, "ssl_capath"s, "ssl_verify"s } )
407 {
408 const std::string & value( template_r.getQueryParam( param ) );
409 if ( ! value.empty() )
410 url_r.setQueryParam( param, value );
411 }
412 return url_r;
413}
414
416 curl_multi_setopt( _parent._multi, CURLMOPT_SOCKETFUNCTION, socketcb );
417 curl_multi_setopt( _parent._multi, CURLMOPT_SOCKETDATA, this );
418 curl_multi_setopt( _parent._multi, CURLMOPT_TIMERFUNCTION, timercb );
419 curl_multi_setopt( _parent._multi, CURLMOPT_TIMERDATA, this );
420}
421
423 curl_multi_setopt( _parent._multi, CURLMOPT_SOCKETFUNCTION, nullptr );
424 curl_multi_setopt( _parent._multi, CURLMOPT_SOCKETDATA, nullptr );
425 curl_multi_setopt( _parent._multi, CURLMOPT_TIMERFUNCTION, nullptr );
426 curl_multi_setopt( _parent._multi, CURLMOPT_TIMERDATA, nullptr );
427}
428
429int CurlPollHelper::socketcb(CURL *easy, curl_socket_t s, int what, CurlPollHelper *userp, void *sockp) {
430 auto it = std::find_if( userp->socks.begin(), userp->socks.end(), [&]( const GPollFD &fd){ return fd.fd == s; });
431 gushort events = 0;
432 if ( what == CURL_POLL_REMOVE ) {
433 if ( it == userp->socks.end() ) {
434 WAR << "Ignoring unknown socket in static_socketcb" << std::endl;
435 return 0;
436 }
437 userp->socks.erase(it);
438 return 0;
439 } else if ( what == CURL_POLL_IN ) {
440 events = G_IO_IN | G_IO_HUP | G_IO_ERR;
441 } else if ( what == CURL_POLL_OUT ) {
442 events = G_IO_OUT | G_IO_ERR;
443 } else if ( what == CURL_POLL_INOUT ) {
444 events = G_IO_IN | G_IO_OUT | G_IO_HUP | G_IO_ERR;
445 }
446
447 if ( it != userp->socks.end() ) {
448 it->events = events;
449 it->revents = 0;
450 } else {
451 userp->socks.push_back(
452 GPollFD{
453 .fd = s,
454 .events = events,
455 .revents = 0
456 }
457 );
458 }
459 return 0;
460}
461
463 if ( !thatPtr )
464 return 0;
465 if ( timeout_ms == -1 )
466 thatPtr->timeout_ms.reset(); // curl wants to delete its timer
467 else
468 thatPtr->timeout_ms = timeout_ms; // maximum time curl wants us to sleep
469 return 0;
470}
471
472CURLMcode internal::CurlPollHelper::handleSocketActions( const std::vector<GPollFD> &actionsFds , int first )
473{
474 for ( size_t sock = first; sock < actionsFds.size(); sock++ ) {
475 const auto &waitFd = actionsFds[sock];
476 if ( waitFd.revents == 0 )
477 continue;
478
479 int ev = 0;
480 if ( (waitFd.revents & G_IO_HUP) == G_IO_HUP
481 || (waitFd.revents & G_IO_IN) == G_IO_IN ) {
482 ev = CURL_CSELECT_IN;
483 }
484 if ( (waitFd.revents & G_IO_OUT) == G_IO_OUT ) {
485 ev |= CURL_CSELECT_OUT;
486 }
487 if ( (waitFd.revents & G_IO_ERR) == G_IO_ERR ) {
488 ev |= CURL_CSELECT_ERR;
489 }
490
491 int runn = 0;
492 CURLMcode mcode = curl_multi_socket_action( _parent._multi, waitFd.fd, ev, &runn );
493 if (mcode != CURLM_OK)
494 return mcode;
495 }
496 return CURLM_OK;
497}
498
500{
501 int handles = 0;
502 return curl_multi_socket_action( _parent._multi, CURL_SOCKET_TIMEOUT, 0, &handles );
503}
504
515CURLcode setCurlRedirProtocols(CURL *curl)
516{
517#if CURLVERSION_AT_LEAST(7,19,4)
518#if CURLVERSION_AT_LEAST(7,85,0)
519 // runtime version might be different from build version
520 if( ::internal::curlVersion() >= CURL_VERSION_BITS(7,85,0) ) {
521 return curl_easy_setopt ( curl, CURLOPT_REDIR_PROTOCOLS_STR, "https" );
522 } else {
523 return curl_easy_setopt ( curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
524 }
525#else
526 return curl_easy_setopt ( curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
527#endif
528#endif // #if CURLVERSION_AT_LEAST(7,19,4)
529 return CURLE_OK;
530}
531
532}
struct _GPollFD GPollFD
Definition ZYppImpl.h:26
const std::string & msg() const
Return the message string provided to the ctor.
Definition Exception.h:196
Url manipulation class.
Definition Url.h:93
void delQueryParams(const std::set< std::string > &params)
remove multiple query parameters at once
Definition Url.cc:869
std::string asString() const
Returns a default string representation of the Url object.
Definition Url.cc:515
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition Url.cc:590
void setFragment(const std::string &fragment, EEncoding eflag=zypp::url::E_DECODED)
Set the fragment string in the URL.
Definition Url.cc:740
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition Url.cc:678
void setPassword(const std::string &pass, EEncoding eflag=zypp::url::E_DECODED)
Set the password in the URL authority.
Definition Url.cc:757
void setPathParams(const std::string &params)
Set the path parameters.
Definition Url.cc:809
void setQueryParam(const std::string &param, const std::string &value)
Set or add value for the specified query parameter.
Definition Url.cc:856
void setUsername(const std::string &user, EEncoding eflag=zypp::url::E_DECODED)
Set the username in the URL authority.
Definition Url.cc:748
std::string getPassword(EEncoding eflag=zypp::url::E_DECODED) const
Returns the password from the URL authority.
Definition Url.cc:598
Wrapper class for stat/lstat.
Definition PathInfo.h:226
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Just inherits Exception to separate media exceptions.
bool useProxyFor(const Url &url_r) const
Return true if enabled and url_r does not match noProxy.
Definition proxyinfo.cc:56
std::string proxy(const Url &url) const
Definition proxyinfo.cc:44
Holds transfer setting.
void setProxy(const std::string &val_r)
proxy to use if it is enabled
void setProxyEnabled(bool enabled)
whether the proxy is used or not
void setUsername(const std::string &val_r)
sets the auth username
void setProxyUsername(const std::string &val_r)
sets the proxy user
void setHeadRequestsAllowed(bool allowed)
set whether HEAD requests are allowed
void setVerifyHostEnabled(bool enabled)
Sets whether to verify host for ssl.
void setClientKeyPath(const Pathname &val_r)
Sets the SSL client key file.
void setClientCertificatePath(const Pathname &val_r)
Sets the SSL client certificate file.
void setPassword(const std::string &val_r)
sets the auth password
void setVerifyPeerEnabled(bool enabled)
Sets whether to verify host for ssl.
void setAnonymousAuth()
sets anonymous authentication (ie: for ftp)
const std::string & username() const
auth username
const std::string & proxyUsername() const
proxy auth username
void setAuthType(const std::string &val_r)
set the allowed authentication types
void setCertificateAuthoritiesPath(const Pathname &val_r)
Sets the SSL certificate authorities path.
void setProxyPassword(const std::string &val_r)
sets the proxy password
void setTimeout(long t)
set the transfer timeout
#define TRANSFER_TIMEOUT_MAX
Definition curlhelper.cc:27
#define EXPLICITLY_NO_PROXY
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
void globalInitCurlOnce()
Definition curlhelper.cc:64
uint curlVersion()
Definition curlhelper.cc:74
zypp::Url propagateQueryParams(zypp::Url url_r, const zypp::Url &template_r)
std::string curlUnEscape(const std::string &text_r)
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
std::string curlEscapedPath(std::string path_r)
CURLcode setCurlRedirProtocols(CURL *curl)
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Url clearQueryString(const Url &url)
void curlEscape(std::string &str_r, const char char_r, const std::string &escaped_r)
int log_curl(CURL *curl, curl_infotype info, char *ptr, size_t len, void *max_lvl)
Definition curlhelper.cc:80
Namespace intended to collect all environment variables we use.
Definition Env.h:23
const long & ZYPP_MEDIA_CURL_DEBUG()
const long& for setting CURLOPT_DEBUGDATA Returns a reference to a static variable,...
Definition curlhelper.cc:36
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition curlhelper.cc:45
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition String.h:1026
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \t", const Trim trim_r=NO_TRIM)
Split line_r into words.
Definition String.h:531
TInt strtonum(const C_Str &str)
Parsing numbers from string.
Url details namespace.
Definition UrlBase.cc:58
@ E_ENCODED
Flag to request encoded string(s).
Definition UrlUtils.h:53
Easy-to use interface to the ZYPP dependency resolver.
std::ostream & hexdumpOn(std::ostream &outs, const unsigned char *ptr, size_t size)
hexdump data on stream
Definition LogTools.h:472
static int socketcb(CURL *easy, curl_socket_t s, int what, CurlPollHelper *userp, void *sockp)
CurlPollHelper(CurlPoll &p)
CURLMcode handleSocketActions(const std::vector< GPollFD > &actionsFds, int first=0)
std::vector< GPollFD > socks
std::optional< long > timeout_ms
static int timercb(CURLM *, long timeout_ms, CurlPollHelper *thatPtr)
Provides API related macros.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition Exception.h:424
#define DBG
Definition Logger.h:99
#define MIL
Definition Logger.h:100
#define WAR
Definition Logger.h:101
#define INT
Definition Logger.h:104