libzypp  17.31.8
Downloader.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
9 
10 #include <fstream>
11 #include <solv/solvversion.h>
12 #include <zypp/base/String.h>
13 #include <zypp/base/LogTools.h>
14 #include <zypp/base/Function.h>
15 #include <zypp/ZConfig.h>
16 
17 #include "Downloader.h"
20 #include <zypp-core/base/UserRequestException>
21 #include <zypp/parser/xml/Reader.h>
23 
24 using namespace zypp::xml;
25 using namespace zypp::parser::yum;
26 
27 namespace zypp
28 {
29 namespace repo
30 {
31 namespace yum
32 {
34  namespace
35  {
36  inline OnMediaLocation loc_with_path_prefix( OnMediaLocation loc_r, const Pathname & prefix_r )
37  {
38  if ( ! prefix_r.empty() && prefix_r != "/" )
39  loc_r.changeFilename( prefix_r / loc_r.filename() );
40  return loc_r;
41  }
42 
43  // search old repository file to run the delta algorithm on
44  Pathname search_deltafile( const Pathname & dir, const Pathname & file )
45  {
47  if ( ! PathInfo(dir).isDir() )
48  return deltafile;
49 
50  // Strip the checksum preceding the file stem so we can look for an
51  // old *-primary.xml which may contain some reusable blocks.
52  std::string base { file.basename() };
53  size_t hypoff = base.find( "-" );
54  if ( hypoff != std::string::npos )
55  base.replace( 0, hypoff + 1, "" );
56 
57  std::list<std::string> retlist;
58  if ( ! filesystem::readdir( retlist, dir, false ) )
59  {
60  for ( const auto & fn : retlist )
61  {
62  if ( str::endsWith( fn, base ) )
63  deltafile = fn;
64  }
65  }
66  if ( !deltafile.empty() )
67  return dir/deltafile;
68 
69  return deltafile;
70  }
71  } // namespace
73 
88  {
89  NON_COPYABLE( Impl );
90  NON_MOVABLE( Impl );
91 
92  Impl( Downloader & downloader_r, MediaSetAccess & media_r, const Pathname & destDir_r )
93  : _downloader { downloader_r }
94  , _media { media_r }
95  , _destDir { destDir_r }
96  {
97  addWantedLocale( ZConfig::instance().textLocale() );
98  for ( const Locale & it : ZConfig::instance().repoRefreshLocales() )
99  addWantedLocale( it );
100  }
101 
106  bool operator()( const OnMediaLocation & loc_r, const std::string & typestr_r )
107  {
108  if ( str::endsWith( typestr_r, "_db" ) )
109  return true; // skip sqlitedb
110 
111  bool zchk { str::endsWith( typestr_r, "_zck" ) };
112 #if defined(LIBSOLVEXT_FEATURE_ZCHUNK_COMPRESSION)
113  const std::string & basetype { zchk ? typestr_r.substr( 0, typestr_r.size()-4 ) : typestr_r };
114 #else
115  if ( zchk )
116  return true; // skip zchunk if not supported by libsolv
117  const std::string & basetype { typestr_r };
118 #endif
119 
120  // filter well known resource types
121  if ( basetype == "other" || basetype == "filelists" )
122  return true; // skip it
123 
124  // filter localized susedata
125  if ( str::startsWith( basetype, "susedata." ) )
126  {
127  // susedata.LANG
128  if ( ! wantLocale( Locale(basetype.c_str()+9) ) )
129  return true; // skip it
130  }
131 
132  // may take it... (prefer zchnk)
133  if ( zchk || !_wantedFiles.count( basetype ) )
134  _wantedFiles[basetype] = loc_r;
135 
136  return true;
137  }
138 
139  void finalize()
140  {
141  // schedule fileS for download
142  for ( const auto & el : _wantedFiles )
143  {
144  const OnMediaLocation & loc { el.second };
145  const OnMediaLocation & loc_with_path { loc_with_path_prefix( loc, _downloader.repoInfo().path() ) };
146  _downloader.enqueueDigested( OnMediaLocation(loc_with_path).setDeltafile( search_deltafile( deltaDir()/"repodata", loc.filename() ) ), FileChecker() );
147  }
148  }
149 
150  private:
151  const Pathname & deltaDir() const
152  { return _downloader._deltaDir; }
153 
154  bool wantLocale( const Locale & locale_r ) const
155  { return _wantedLocales.count( locale_r ); }
156 
157  void addWantedLocale( Locale locale_r )
158  {
159  while ( locale_r )
160  {
161  _wantedLocales.insert( locale_r );
162  locale_r = locale_r.fallback();
163  }
164  }
165 
166  private:
170 
172  std::map<std::string,OnMediaLocation> _wantedFiles;
173  };
174 
176  //
177  // class Downloader
178  //
180 
181  Downloader::Downloader( const RepoInfo & info_r, const Pathname & deltaDir_r )
182  : repo::Downloader { info_r}
183  , _deltaDir { deltaDir_r }
184  {}
185 
186  void Downloader::download( MediaSetAccess & media_r, const Pathname & destDir_r, const ProgressData::ReceiverFnc & progress_r )
187  {
188  downloadMediaInfo( destDir_r, media_r );
189 
190  Pathname masterIndex { repoInfo().path() / "/repodata/repomd.xml" };
191  defaultDownloadMasterIndex( media_r, destDir_r, masterIndex );
192 
193  //enable precache
194  setMediaSetAccess( media_r );
195 
196  // setup parser
197  Impl pimpl( *this, media_r, destDir_r );
198  RepomdFileReader( destDir_r / masterIndex, std::ref(pimpl) );
199  pimpl.finalize();
200 
201  // ready, go!
202  start( destDir_r );
203  }
204 
206  {
207  const auto & ri = repoInfo();
208  RepoStatus ret { media_r.provideOptionalFile( ri.path() / "/repodata/repomd.xml" ) };
209  if ( !ret.empty() && ri.requireStatusWithMediaFile() ) // else: mandatory master index is missing
210  ret = ret && RepoStatus( media_r.provideOptionalFile( "/media.1/media" ) );
211  // else: mandatory master index is missing -> stay empty
212  return ret;
213  }
214 } // namespace yum
215 } // namespace repo
216 } // namespace zypp
Pathname path() const
Repository path.
Definition: RepoInfo.cc:722
void addWantedLocale(Locale locale_r)
Definition: Downloader.cc:157
void defaultDownloadMasterIndex(MediaSetAccess &media_r, const Pathname &destdir_r, const Pathname &masterIndex_r)
Common workflow downloading a (signed) master index file.
Definition: Downloader.cc:141
Pathname deltafile
Describes a resource file located on a medium.
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:832
Pathname provideOptionalFile(const Pathname &file, unsigned media_nr=1)
Provides an optional file from media media_nr.
Helper filtering the files offered by a RepomdFileReader.
Definition: Downloader.cc:87
Locale fallback() const
Return the fallback locale for this locale, if no fallback exists the empty Locale::noCode.
Definition: Locale.cc:208
void downloadMediaInfo(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progressrcv)
Downloads the media info (/media.1) to a local directory.
What is known about a repository.
Definition: RepoInfo.h:71
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:128
bool operator()(const OnMediaLocation &loc_r, const std::string &typestr_r)
The callback invoked by the RepomdFileReader.
Definition: Downloader.cc:106
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
Definition: progressdata.h:140
LocaleSet _wantedLocales
Locales do download.
Definition: Downloader.cc:171
void setMediaSetAccess(MediaSetAccess &media)
Sets the media set access that will be used to precache and to download the files when start is calle...
Definition: Fetcher.cc:903
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir The media has to be provides with setMediaSetA...
Definition: Fetcher.cc:908
std::map< std::string, OnMediaLocation > _wantedFiles
Definition: Downloader.cc:172
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
const Pathname & deltaDir() const
Definition: Downloader.cc:151
void download(MediaSetAccess &media_r, const Pathname &destDir_r, const ProgressData::ReceiverFnc &progress_r=ProgressData::ReceiverFnc()) override
Download metadata to a local directory.
Definition: Downloader.cc:186
Impl(Downloader &downloader_r, MediaSetAccess &media_r, const Pathname &destDir_r)
Definition: Downloader.cc:92
RepoStatus status(MediaSetAccess &media_r) override
Status of the remote repository.
Definition: Downloader.cc:205
bool startsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasPrefix
Definition: String.h:1085
const RepoInfo & repoInfo() const
Definition: Downloader.h:63
static Pathname search_deltafile(const Pathname &dir, const Pathname &file)
Definition: Downloader.cc:45
const Pathname & filename() const
The path to the resource on the medium.
bool endsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasSuffix
Definition: String.h:1092
bool wantLocale(const Locale &locale_r) const
Definition: Downloader.cc:154
&#39;Language[_Country]&#39; codes.
Definition: Locale.h:49
int readdir(std::list< std::string > &retlist_r, const Pathname &path_r, bool dots_r)
Return content of directory via retlist.
Definition: PathInfo.cc:605
#define NON_MOVABLE(CLASS)
Delete move ctor and move assign.
Definition: Easy.h:69
Reads through a repomd.xml file and collects type, location, checksum and other data about metadata f...
#define NON_COPYABLE(CLASS)
Delete copy ctor and copy assign.
Definition: Easy.h:59
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:220
Interface of repomd.xml file reader.
Track changing files or directories.
Definition: RepoStatus.h:40
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
Definition: FileChecker.h:29
OnMediaLocation & changeFilename(Pathname filename_r)
Individual manipulation of filename (prefer setLocation).
Downloader for YUM (rpm-nmd) repositories Encapsulates all the knowledge of which files have to be do...
Definition: Downloader.h:40
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
std::unordered_set< Locale > LocaleSet
Definition: Locale.h:27