libzypp  17.37.5
MediaCurl2.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
13 #include <iostream>
14 
15 #include <zypp/base/Logger.h>
16 #include <zypp/ExternalProgram.h>
17 #include <zypp/base/String.h>
18 #include <zypp/base/Gettext.h>
19 #include <zypp-core/parser/Sysconfig>
20 
21 #include <zypp/media/MediaCurl2.h>
22 
25 #include <zypp-curl/ProxyInfo>
26 #include <zypp-curl/CurlConfig>
28 #include <zypp/Target.h>
29 #include <zypp/ZYppFactory.h>
30 #include <zypp/ZConfig.h>
31 #include <zypp/zypp_detail/ZYppImpl.h> // for zypp_poll
32 
35 
36 #include <cstdlib>
37 #include <sys/types.h>
38 #include <sys/stat.h>
39 #include <sys/mount.h>
40 #include <dirent.h>
41 #include <unistd.h>
42 #include <glib.h>
43 
47 
48 #ifdef ENABLE_ZCHUNK_COMPRESSION
50 #endif
51 
52 
59 using std::endl;
60 
61 using namespace internal;
62 using namespace zypp::base;
63 
64 namespace zypp {
65 
66  namespace media {
67 
68  MediaCurl2::MediaCurl2(const MediaUrl &url_r, const std::vector<MediaUrl> &mirrors_r,
69  const Pathname & attach_point_hint_r )
70  : MediaNetworkCommonHandler( url_r, mirrors_r, attach_point_hint_r,
71  "/", // urlpath at attachpoint
72  true ) // does_download
73  , _executor( std::make_shared<internal::MediaNetworkRequestExecutor>() )
74  {
75 
76  MIL << "MediaCurl2::MediaCurl2(" << url_r.url() << ", " << attach_point_hint_r << ")" << endl;
77 
78  if( !attachPoint().empty())
79  {
80  PathInfo ainfo(attachPoint());
81  Pathname apath(attachPoint() + "XXXXXX");
82  char *atemp = ::strdup( apath.asString().c_str());
83  char *atest = NULL;
84  if( !ainfo.isDir() || !ainfo.userMayRWX() ||
85  atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
86  {
87  WAR << "attach point " << ainfo.path()
88  << " is not useable for " << url_r.url().getScheme() << endl;
89  setAttachPoint("", true);
90  }
91  else if( atest != NULL)
92  ::rmdir(atest);
93 
94  if( atemp != NULL)
95  ::free(atemp);
96  }
97  }
98 
100 
101  void MediaCurl2::checkProtocol(const Url &url) const
102  {
103  if ( !zyppng::NetworkRequestDispatcher::supportsProtocol ( url ) )
104  {
105  std::string msg("Unsupported protocol '");
106  msg += url.getScheme();
107  msg += "'";
108  ZYPP_THROW(MediaBadUrlException(_url.url(), msg));
109  }
110  }
111 
113  {
114  // clear effective settings
115  _mirrorSettings.clear();
116  }
117 
119 
120  void MediaCurl2::releaseFrom( const std::string & ejectDev )
121  {
122  disconnect();
123  }
124 
126 
127  void MediaCurl2::getFileCopy( const OnMediaLocation & srcFile , const Pathname & target ) const
128  {
129 
130  const auto &filename = srcFile.filename();
131 
132  // Optional files will send no report until data are actually received (we know it exists).
133  OptionalDownloadProgressReport reportfilter( srcFile.optional() );
135 
136  for ( unsigned mirr = 0; mirr < _urls.size(); ++mirr ) {
137 
138  MIL << "Trying to fetch file " << srcFile << " with mirror " << _urls[mirr].url() << std::endl;
139  Url fileurl(getFileUrl(mirr, filename));
140 
141  try
142  {
143  if(!_urls[mirr].url().isValid())
145 
146  if(_urls[mirr].url().getHost().empty())
148 
149 
150  Pathname dest = target.absolutename();
151  if( assert_dir( dest.dirname() ) ) {
152  DBG << "assert_dir " << dest.dirname() << " failed" << endl;
153  ZYPP_THROW( MediaSystemException(fileurl, "System error on " + dest.dirname().asString()) );
154  }
155 
156  ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) }; {
157  AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
158  if( ! buf ) {
159  ERR << "out of memory for temp file name" << endl;
160  ZYPP_THROW(MediaSystemException(fileurl, "out of memory for temp file name"));
161  }
162 
163  AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
164  if( tmp_fd == -1 ) {
165  ERR << "mkstemp failed for file '" << destNew << "'" << endl;
167  }
168  destNew = ManagedFile( (*buf), filesystem::unlink );
169  }
170 
171  DBG << "dest: " << dest << endl;
172  DBG << "temp: " << destNew << endl;
173  #if 0
174  Not implemented here yet because NetworkRequest can not do IFMODSINCE yet
175  // set IFMODSINCE time condition (no download if not modified)
176  if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
177  {
178  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
179  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
180  }
181  else
182  {
183  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
184  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
185  }
186  #endif
187 
188  DBG << srcFile.filename().asString() << endl;
189  DBG << "URL: " << fileurl.asString() << endl;
190 
191  // Use URL without options and without username and passwd
192  // (some proxies dislike them in the URL).
193  // Curl seems to need the just scheme, hostname and a path;
194  // the rest was already passed as curl options (in attachTo).
195  Url curlUrl( clearQueryString(fileurl) );
196 
197  RequestData r;
198  r._mirrorIdx = mirr;
199  r._req = std::make_shared<zyppng::NetworkRequest>( curlUrl, destNew, zyppng::NetworkRequest::WriteShared /*do not truncate*/ );
200  r._req->transferSettings() = _mirrorSettings[mirr];
201  r._req->setExpectedFileSize ( srcFile.downloadSize () );
202 
203  bool done = false;
204  #ifdef ENABLE_ZCHUNK_COMPRESSION
205  done = const_cast<MediaCurl2*>(this)->tryZchunk(r, srcFile, destNew, report);
206  #endif
207  if ( !done ) {
208  r._req->resetRequestRanges();
209  const_cast<MediaCurl2 *>(this)->executeRequest ( r, &report );
210  }
211 
212  #if 0
213  Also disabled IFMODSINCE code, see above while not yet implemented here
214  #if CURLVERSION_AT_LEAST(7,19,4)
215  // bnc#692260: If the client sends a request with an If-Modified-Since header
216  // with a future date for the server, the server may respond 200 sending a
217  // zero size file.
218  // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
219  if ( ftell(file) == 0 && ret == 0 )
220  {
221  long httpReturnCode = 33;
222  if ( curl_easy_getinfo( _curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
223  {
224  long conditionUnmet = 33;
225  if ( curl_easy_getinfo( _curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
226  {
227  WAR << "TIMECONDITION unmet - retry without." << endl;
228  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
229  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
230  ret = executeCurl();
231  }
232  }
233  }
234  #endif
235  #endif
236 
237 
238  // apply umask
239  if ( ::chmod( destNew->c_str(), filesystem::applyUmaskTo( 0644 ) ) )
240  {
241  ERR << "Failed to chmod file " << destNew << endl;
242  }
243 
244  // move the temp file into dest
245  if ( rename( destNew, dest ) != 0 ) {
246  ERR << "Rename failed" << endl;
248  }
249  destNew.resetDispose(); // no more need to unlink it
250 
251  DBG << "done: " << PathInfo(dest) << endl;
252 
253  break; // success!
254  }
255  catch (MediaException & excpt_r)
256  {
257  // check if we can retry on the next mirror
258  if( !canTryNextMirror ( excpt_r ) || ( mirr+1 >= _urls.size() ) ) {
259  // rewrite the exception to contain the correct pathname and url
260  // the executeRequest implementation just emits the full Url in the exception
261  if ( typeid(excpt_r) == typeid( MediaFileNotFoundException ) ) {
262  ZYPP_CAUGHT(excpt_r);
263  ZYPP_THROW( MediaFileNotFoundException( url(), filename ) );
264  }
265  ZYPP_RETHROW(excpt_r);
266  }
267  ZYPP_CAUGHT(excpt_r);
268  continue;
269  }
270  }
271  }
272 
273  bool MediaCurl2::getDoesFileExist( const Pathname & filename ) const
274  {
275  DBG << filename.asString() << endl;
276 
277  std::exception_ptr lastErr;
278  for ( unsigned mirr = 0; mirr < _urls.size(); ++mirr ) {
279  if( !_urls[mirr].url().isValid() )
281 
282  if( _urls[mirr].url().getHost().empty() )
284 
285  Url url(getFileUrl(mirr, filename));
286 
287  DBG << "URL: " << url.asString() << endl;
288 
289  // Use URL without options and without username and passwd
290  // (some proxies dislike them in the URL).
291  // Curl seems to need the just scheme, hostname and a path;
292  // the rest was already passed as curl options (in attachTo).
293  Url curlUrl( clearQueryString(url) );
294 
295  RequestData r;
296  r._mirrorIdx = mirr;
297  r._req = std::make_shared<zyppng::NetworkRequest>( curlUrl, "/dev/null" );
298  r._req->setOptions ( zyppng::NetworkRequest::HeadRequest ); // just check for existance
299  r._req->transferSettings() = _mirrorSettings[mirr];
300 
301  // as we are not having user interaction, the user can't cancel
302  // the file existence checking, a callback or timeout return code
303  // will be always a timeout.
304  try {
305  const_cast<MediaCurl2*>(this)->executeRequest ( r );
306 
307  } catch ( const MediaException &e ) {
308  // check if we can retry on the next mirror
309  if( !canTryNextMirror ( e ) ) {
310  ZYPP_RETHROW(e);
311  }
312  lastErr = ZYPP_FWD_CURRENT_EXCPT();
313  continue;
314  }
315  // exists
316  return ( !r._req->hasError() );
317  }
318 
319  if ( lastErr ) {
320  try {
321  ZYPP_RETHROW (lastErr);
322  } catch ( const MediaFileNotFoundException &e ) {
323  // if the file did not exist then we can return false
324  return false;
325  }
326  }
327  // we probably never had mirrors, otherwise we either had an error or a success.
328  return false;
329  }
330 
332  {
333 #ifdef ENABLE_ZCHUNK_COMPRESSION
334 
335  // HERE add zchunk logic if required
336  if ( !srcFile.deltafile().empty()
338  && srcFile.headerSize () > 0 ) {
339 
340  // first fetch the zck header
341  std::optional<zypp::Digest> digest;
342  UByteArray sum;
343 
344  const auto &headerSum = srcFile.headerChecksum();
345  if ( !headerSum.empty () ) {
346  digest = zypp::Digest();
347  if ( !digest->create( headerSum.type() ) ) {
348  ERR << "Unknown header checksum type " << headerSum.type() << std::endl;
349  return false;
350  }
351  sum = zypp::Digest::hexStringToUByteArray( headerSum.checksum() );
352  }
353 
354  reqData._req->addRequestRange( 0, srcFile.headerSize(), std::move(digest), sum );
355  executeRequest ( reqData, nullptr );
356 
357  reqData._req->resetRequestRanges();
358 
359  auto res = zyppng::ZckHelper::prepareZck( srcFile.deltafile(), target, srcFile.downloadSize() );
360  switch(res._code) {
362  ERR << "Failed to setup zchunk because of: " << res._message << std::endl;
363  return false;
364  }
366  return true; // already done
368  ZYPP_THROW( MediaFileSizeExceededException( reqData._req->url(), srcFile.downloadSize(), res._message ));
370  break;
371  }
372 
373  for ( const auto &block : res._blocks ) {
374  if ( block._checksum.size() && block._chksumtype.size() ) {
375  std::optional<zypp::Digest> dig = zypp::Digest();
376  if ( !dig->create( block._chksumtype ) ) {
377  WAR_MEDIA << "Trying to create Digest with chksum type " << block._chksumtype << " failed " << std::endl;
378  return false;
379  }
380 
382  DBG_MEDIA << "Starting block " << block._start << " with checksum " << zypp::Digest::digestVectorToString( block._checksum ) << "." << std::endl;
383  reqData._req->addRequestRange( block._start, block._len, std::move(dig), block._checksum, {}, block._relevantDigestLen, block._chksumPad );
384  }
385  };
386 
387  executeRequest ( reqData, &report );
388 
389  //we might have the file ready
390  std::string err;
391  if ( !zyppng::ZckHelper::validateZckFile( target, err) ) {
392  ERR << "ZCK failed with error: " << err << std::endl;
393  return false;
394  }
395  return true;
396  }
397 #endif
398  return false;
399  }
400 
402  {
403  const auto &authCb = [&]( const zypp::Url &, TransferSettings &settings, const std::string & availAuthTypes, bool firstTry, bool &canContinue ) {
404  if ( authenticate( _urls[reqData._mirrorIdx].url(), _mirrorSettings[reqData._mirrorIdx], availAuthTypes, firstTry ) ) {
405  settings = _mirrorSettings[reqData._mirrorIdx];
406  canContinue = true;
407  return;
408  }
409  canContinue = false;
410  };
411 
412  auto conn = _executor->sigAuthRequired().connect (authCb);
413  zypp_defer {
414  conn.disconnect ();
415  };
416 
417  _executor->executeRequest ( reqData._req, report );
418  }
419  } // namespace media
420 } // namespace zypp
421 //
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:551
Interface to gettext.
#define MIL
Definition: Logger.h:100
#define DBG_MEDIA
Definition: mediadebug_p.h:28
void checkProtocol(const Url &url) const override
check the url is supported by the curl library
Definition: MediaCurl2.cc:101
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:324
const ByteCount & headerSize() const
The size of the header prepending the resource (e.g.
const Pathname & path() const
Return current Pathname.
Definition: PathInfo.h:251
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:459
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl2.h:43
Describes a resource file located on a medium.
bool getDoesFileExist(const Pathname &filename) const override
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly, or user cancels the operation.
Definition: MediaCurl2.cc:273
Compute Message Digests (MD5, SHA1 etc)
Definition: Digest.h:37
void getFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename) const override
Definition: MediaCurl2.cc:127
Holds transfer setting.
const MediaUrl _url
Primary Url.
Definition: MediaHandler.h:118
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:175
static bool canTryNextMirror(const Excpt &excpt_r)
int chmod(const Pathname &path, mode_t mode)
Like &#39;chmod&#39;.
Definition: PathInfo.cc:1097
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
Definition: Arch.h:363
std::vector< TransferSettings > _mirrorSettings
void disconnectFrom() override
Definition: MediaCurl2.cc:112
bool authenticate(const Url &url, TransferSettings &settings, const std::string &availAuthTypes, bool firstTry)
AutoDispose<int> calling ::close
Definition: AutoDispose.h:309
#define zypp_defer
Definition: AutoDispose.h:293
#define ERR
Definition: Logger.h:102
bool tryZchunk(RequestData &reqData, const OnMediaLocation &srcFile, const Pathname &target, callback::SendReport< DownloadProgressReport > &report)
Definition: MediaCurl2.cc:331
static bool isZchunkFile(const zypp::Pathname &file)
Definition: zckhelper.cc:21
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition: ManagedFile.h:27
static std::string digestVectorToString(const UByteArray &vec)
get hex string representation of the digest vector given as parameter
Definition: Digest.cc:244
bool empty() const
Test for an empty path.
Definition: Pathname.h:116
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:479
const std::vector< MediaUrl > _urls
All mirrors including the primary.
Definition: MediaHandler.h:113
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:515
Bottleneck filtering all DownloadProgressReport issued from Media[Muli]Curl.
const std::string & asString() const
String representation.
Definition: Pathname.h:93
Just inherits Exception to separate media exceptions.
const ByteCount & downloadSize() const
The size of the resource on the server.
void disconnect()
Use concrete handler to isconnect media.
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:126
#define WAR
Definition: Logger.h:101
const long & ZYPP_MEDIA_CURL_DEBUG()
const long& for setting CURLOPT_DEBUGDATA Returns a reference to a static variable, so it&#39;s safe to pass ...
Definition: curlhelper.cc:36
void executeRequest(RequestData &reqData, callback::SendReport< DownloadProgressReport > *report=nullptr)
Definition: MediaCurl2.cc:401
zyppng::NetworkRequestRef _req
Definition: MediaCurl2.h:96
bool isValid() const
Verifies the Url.
Definition: Url.cc:507
const Pathname & filename() const
The path to the resource on the medium.
int unlink(const Pathname &path)
Like &#39;unlink&#39;.
Definition: PathInfo.cc:705
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:475
const Pathname & deltafile() const
The existing deltafile that can be used to reduce download size ( zchunk or metalink ) ...
#define WAR_MEDIA
Definition: mediadebug_p.h:30
Pathname absolutename() const
Return this path, adding a leading &#39;/&#39; if relative.
Definition: Pathname.h:141
Pathname attachPoint() const
Return the currently used attach point.
Url url() const
Primary Url used.
Definition: MediaHandler.h:509
Url getFileUrl(int mirrorIdx, const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
static bool validateZckFile(const zypp::Pathname &file, std::string &error)
Definition: zckhelper.cc:169
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:606
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:225
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:806
#define ZYPP_FWD_CURRENT_EXCPT()
Drops a logline and returns the current Exception as a std::exception_ptr.
Definition: Exception.h:471
int rename(const Pathname &oldpath, const Pathname &newpath)
Like &#39;rename&#39;.
Definition: PathInfo.cc:747
Easy-to use interface to the ZYPP dependency resolver.
Definition: Application.cc:19
int rmdir(const Pathname &path)
Like &#39;rmdir&#39;.
Definition: PathInfo.cc:371
bool optional() const
Whether this is an optional resource.
bool userMayRWX() const
Definition: PathInfo.h:361
Url manipulation class.
Definition: Url.h:92
void releaseFrom(const std::string &ejectDev) override
Call concrete handler to release the media.
Definition: MediaCurl2.cc:120
internal::MediaNetworkRequestExecutorRef _executor
Definition: MediaCurl2.h:103
#define DBG
Definition: Logger.h:99
const CheckSum & headerChecksum() const
The checksum of the header prepending the resource (e.g.
static PrepareResult prepareZck(const zypp::Pathname &delta, const zypp::Pathname &target, const zypp::ByteCount &expectedFileSize)
Definition: zckhelper.cc:34