yum.yumRepo

1 # This program is free software; you can redistribute it and/or modify 2 # it under the terms of the GNU General Public License as published by 3 # the Free Software Foundation; either version 2 of the License, or 4 # (at your option) any later version. 5 # 6 # This program is distributed in the hope that it will be useful, 7 # but WITHOUT ANY WARRANTY; without even the implied warranty of 8 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 # GNU Library General Public License for more details. 10 # 11 # You should have received a copy of the GNU General Public License 12 # along with this program; if not, write to the Free Software 13 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 14 # Copyright 2005 Duke University 15 # Copyright 2007 Red Hat 16 import os 17 import re 18 import time 19 import types 20 import urlparse 21 urlparse.uses_fragment.append("media") 22 import gzip 23 24 import Errors 25 from urlgrabber.grabber import URLGrabber 26 from urlgrabber.grabber import default_grabber 27 import urlgrabber.mirror 28 from urlgrabber.grabber import URLGrabError 29 import repoMDObject 30 import packageSack 31 from repos import Repository 32 import parser 33 import sqlitecachec 34 import sqlitesack 35 from yum import config 36 from yum import misc 37 from yum import comps 38 from constants import * 39 import metalink 40 41 import logging 42 import logginglevels 43 44 import warnings 45 46 import glob 47 import shutil 48 import stat 49 import errno 50 import tempfile 51 52 # If you want yum to _always_ check the MD .sqlite files then set this to 53 # False (this doesn't affect .xml files or .sqilte files derived from them). 54 # With this as True yum will only check when a new repomd.xml or 55 # new MD is downloaded. 56 # Note that with atomic MD, we can't have old MD lying around anymore so 57 # the only way we need this check is if someone does something like: 58 # cp primary.sqlite /var/cache/yum/blah 59 # ...at which point you lose. 60 skip_old_DBMD_check = True 61 62 warnings.simplefilter("ignore", Errors.YumFutureDeprecationWarning) 63 64 logger = logging.getLogger("yum.Repos") 65 verbose_logger = logging.getLogger("yum.verbose.Repos")

66 67 -class YumPackageSack(packageSack.PackageSack):

68 """imports/handles package objects from an mdcache dict object"""

69 - def __init__(self, packageClass):

70 packageSack.PackageSack.__init__(self) 71 self.pc = packageClass 72 self.added = {}

73

74 - def __del__(self):

75 self.close()

76

77 - def close(self):

78 self.added = {}

79

80 - def addDict(self, repo, datatype, dataobj, callback=None):

81 if self.added.has_key(repo): 82 if datatype in self.added[repo]: 83 return 84 85 total = len(dataobj) 86 if datatype == 'metadata': 87 current = 0 88 for pkgid in dataobj: 89 current += 1 90 if callback: callback.progressbar(current, total, repo) 91 pkgdict = dataobj[pkgid] 92 po = self.pc(repo, pkgdict) 93 po.id = pkgid 94 self._addToDictAsList(self.pkgsByID, pkgid, po) 95 self.addPackage(po) 96 97 if not self.added.has_key(repo): 98 self.added[repo] = [] 99 self.added[repo].append('metadata') 100 # indexes will need to be rebuilt 101 self.indexesBuilt = 0 102 103 elif datatype in ['filelists', 'otherdata']: 104 if self.added.has_key(repo): 105 if 'metadata' not in self.added[repo]: 106 raise Errors.RepoError, '%s md for %s imported before primary' \ 107 % (datatype, repo.id) 108 current = 0 109 for pkgid in dataobj: 110 current += 1 111 if callback: callback.progressbar(current, total, repo) 112 pkgdict = dataobj[pkgid] 113 if self.pkgsByID.has_key(pkgid): 114 for po in self.pkgsByID[pkgid]: 115 po.importFromDict(pkgdict) 116 117 self.added[repo].append(datatype) 118 # indexes will need to be rebuilt 119 self.indexesBuilt = 0 120 else: 121 # umm, wtf? 122 pass

123

124 - def populate(self, repo, mdtype='metadata', callback=None, cacheonly=0):

125 if mdtype == 'all': 126 data = ['metadata', 'filelists', 'otherdata'] 127 else: 128 data = [ mdtype ] 129 130 if not hasattr(repo, 'cacheHandler'): 131 repo.cacheHandler = sqlitecachec.RepodataParserSqlite( 132 storedir=repo.cachedir, 133 repoid=repo.id, 134 callback=callback, 135 ) 136 for item in data: 137 if self.added.has_key(repo): 138 if item in self.added[repo]: 139 continue 140 141 db_fn = None 142 143 if item == 'metadata': 144 mydbtype = 'primary_db' 145 mymdtype = 'primary' 146 repo_get_function = repo.getPrimaryXML 147 repo_cache_function = repo.cacheHandler.getPrimary 148 149 elif item == 'filelists': 150 mydbtype = 'filelists_db' 151 mymdtype = 'filelists' 152 repo_get_function = repo.getFileListsXML 153 repo_cache_function = repo.cacheHandler.getFilelists 154 155 elif item == 'otherdata': 156 mydbtype = 'other_db' 157 mymdtype = 'other' 158 repo_get_function = repo.getOtherXML 159 repo_cache_function = repo.cacheHandler.getOtherdata 160 161 else: 162 continue 163 164 if self._check_db_version(repo, mydbtype): 165 # see if we have the uncompressed db and check it's checksum vs the openchecksum 166 # if not download the bz2 file 167 # decompress it 168 # unlink it 169 170 db_un_fn = self._check_uncompressed_db(repo, mydbtype) 171 if not db_un_fn: 172 db_fn = repo._retrieveMD(mydbtype) 173 if db_fn: 174 db_un_fn = db_fn.replace('.bz2', '') 175 if not repo.cache: 176 misc.bunzipFile(db_fn, db_un_fn) 177 misc.unlink_f(db_fn) 178 db_un_fn = self._check_uncompressed_db(repo, mydbtype) 179 180 dobj = repo.cacheHandler.open_database(db_un_fn) 181 182 else: 183 repo._xml2sqlite_local = True 184 xml = repo_get_function() 185 xmldata = repo.repoXML.getData(mymdtype) 186 (ctype, csum) = xmldata.checksum 187 dobj = repo_cache_function(xml, csum) 188 189 if not cacheonly: 190 self.addDict(repo, item, dobj, callback) 191 del dobj 192 193 194 # get rid of all this stuff we don't need now 195 del repo.cacheHandler

196

197 - def _check_uncompressed_db(self, repo, mdtype):

198 """return file name of uncompressed db is good, None if not""" 199 mydbdata = repo.repoXML.getData(mdtype) 200 (r_base, remote) = mydbdata.location 201 fname = os.path.basename(remote) 202 bz2_fn = repo.cachedir + '/' + fname 203 db_un_fn = bz2_fn.replace('.bz2', '') 204 205 result = None 206 207 repo._preload_md_from_system_cache(os.path.basename(db_un_fn)) 208 if os.path.exists(db_un_fn): 209 if skip_old_DBMD_check and repo._using_old_MD: 210 return db_un_fn 211 212 try: 213 repo.checkMD(db_un_fn, mdtype, openchecksum=True) 214 except URLGrabError: 215 if not repo.cache: 216 misc.unlink_f(db_un_fn) 217 else: 218 result = db_un_fn 219 220 return result

221

222 - def _check_db_version(self, repo, mdtype):

223 return repo._check_db_version(mdtype)

224

225 -class YumRepository(Repository, config.RepoConf):

226 """ 227 This is an actual repository object 228 229 Configuration attributes are pulled in from config.RepoConf. 230 """ 231

232 - def __init__(self, repoid):

233 config.RepoConf.__init__(self) 234 Repository.__init__(self, repoid) 235 236 self.repofile = None 237 self.mirrorurls = [] 238 self._urls = [] 239 self.enablegroups = 0 240 self.groupsfilename = 'yumgroups.xml' # something some freaks might 241 # eventually want 242 self.repoMDFile = 'repodata/repomd.xml' 243 self._repoXML = None 244 self._using_old_MD = None 245 self._oldRepoMDData = {} 246 self.cache = 0 247 self.mirrorlistparsed = 0 248 self.yumvar = {} # empty dict of yumvariables for $string replacement 249 self._proxy_dict = {} 250 self.metadata_cookie_fn = 'cachecookie' 251 self._metadataCurrent = None 252 self._metalink = None 253 self.groups_added = False 254 self.http_headers = {} 255 self.repo_config_age = 0 # if we're a repo not from a file then the 256 # config is very, very old 257 # throw in some stubs for things that will be set by the config class 258 self.basecachedir = "" 259 self.cost = 1000 260 self.copy_local = 0 261 # holder for stuff we've grabbed 262 self.retrieved = { 'primary':0, 'filelists':0, 'other':0, 'group':0, 263 'updateinfo':0, 'prestodelta' : 0} 264 265 # callbacks 266 self.callback = None # for the grabber 267 self.failure_obj = None 268 self.mirror_failure_obj = None 269 self.interrupt_callback = None 270 self._callbacks_changed = False 271 272 # callback function for handling media 273 self.mediafunc = None 274 275 # callbacks for gpg key importing and confirmation 276 self.gpg_import_func = None 277 self.confirm_func = None 278 279 # The reason we want to turn this off are things like repoids 280 # called "tmp" in repoquery --repofrompath and/or new1/old1 in repodiff. 281 self.timestamp_check = True 282 283 self._sack = None 284 285 self._grabfunc = None 286 self._grab = None

287

288 - def __cmp__(self, other):

289 """ Sort yum repos. by cost, and then by alphanumeric on their id. """ 290 if other is None: 291 return 1 292 if hasattr(other, 'cost'): 293 ocost = other.cost 294 else: 295 ocost = 1000 296 ret = cmp(self.cost, ocost) 297 if ret: 298 return ret 299 return cmp(self.id, other.id)

300

301 - def _getSack(self):

302 # FIXME: Note that having the repo hold the sack, which holds "repos" 303 # is not only confusing but creates a circular dep. 304 # Atm. we don't leak memory because RepoStorage.close() is called, 305 # which calls repo.close() which calls sack.close() which removes the 306 # repos from the sack ... thus. breaking the cycle. 307 if self._sack is None: 308 self._sack = sqlitesack.YumSqlitePackageSack( 309 sqlitesack.YumAvailablePackageSqlite) 310 return self._sack

311 sack = property(_getSack) 312

313 - def close(self):

314 if self._sack is not None: 315 self.sack.close() 316 Repository.close(self)

317

318 - def _resetSack(self):

319 self._sack = None

320

321 - def __getProxyDict(self):

322 self.doProxyDict() 323 if self._proxy_dict: 324 return self._proxy_dict 325 return None

326 327 # consistent access to how proxy information should look (and ensuring 328 # that it's actually determined for the repo) 329 proxy_dict = property(__getProxyDict) 330

331 - def getPackageSack(self):

332 """Returns the instance of this repository's package sack.""" 333 return self.sack

334 335

336 - def ready(self):

337 """Returns true if this repository is setup and ready for use.""" 338 if hasattr(self, 'metadata_cookie'): 339 return self.repoXML is not None 340 return False

341 342

343 - def getGroupLocation(self):

344 """Returns the location of the group.""" 345 if 'group_gz' in self.repoXML.fileTypes(): 346 thisdata = self.repoXML.getData('group_gz') 347 else: 348 thisdata = self.repoXML.getData('group') 349 return thisdata.location

350

351 - def __str__(self):

352 return self.id

353

354 - def _checksum(self, sumtype, file, CHUNK=2**16, checksum_can_fail=False, 355 datasize=None):

356 """takes filename, hand back Checksum of it 357 sumtype = md5 or sha 358 filename = /path/to/file 359 CHUNK=65536 by default""" 360 try: 361 return misc.checksum(sumtype, file, CHUNK, datasize) 362 except (Errors.MiscError, EnvironmentError), e: 363 if checksum_can_fail: 364 return None 365 raise Errors.RepoError, 'Error opening file for checksum: %s' % e

366

367 - def dump(self):

368 output = '[%s]\n' % self.id 369 # we exclude all vars which start with _ or are in this list: 370 excluded_vars = ('mediafunc', 'sack', 'metalink_data', 'grab', 371 'grabfunc', 'repoXML', 'cfg', 'retrieved', 372 'mirrorlistparsed', 'gpg_import_func', 'failure_obj', 373 'callback', 'confirm_func', 'groups_added', 374 'interrupt_callback', 'id', 'mirror_failure_obj', 375 'repo_config_age', 'groupsfilename', 'copy_local', 376 'basecachedir', 'http_headers', 'metadata_cookie', 377 'metadata_cookie_fn', 'quick_enable_disable', 378 'repoMDFile', 'timestamp_check', 'urls', 'mirrorurls', 379 'yumvar', 'repofile') 380 for attr in dir(self): 381 if attr.startswith('_'): 382 continue 383 if attr in excluded_vars: 384 continue 385 if isinstance(getattr(self, attr), types.MethodType): 386 continue 387 res = getattr(self, attr) 388 if not res: 389 res = '' 390 if type(res) == types.ListType: 391 res = ',\n '.join(res) 392 output = output + '%s = %s\n' % (attr, res) 393 394 return output

395

396 - def enablePersistent(self):

397 """Persistently enables this repository.""" 398 self.enable() 399 try: 400 config.writeRawRepoFile(self,only=['enabled']) 401 except IOError, e: 402 if e.errno == errno.EACCES: 403 logger.warning(e) 404 else: 405 raise IOError, str(e)

406

407 - def disablePersistent(self):

408 """Persistently disables this repository.""" 409 self.disable() 410 try: 411 config.writeRawRepoFile(self,only=['enabled']) 412 except IOError, e: 413 if e.errno == errno.EACCES: 414 logger.warning(e) 415 else: 416 raise IOError, str(e)

417

418 - def check(self):

419 """self-check the repo information - if we don't have enough to move 420 on then raise a repo error""" 421 if len(self._urls) < 1 and not self.mediaid: 422 raise Errors.RepoError, \ 423 'Cannot find a valid baseurl for repo: %s' % self.id

424

425 - def doProxyDict(self):

426 if self._proxy_dict: 427 return 428 429 self._proxy_dict = {} # zap it 430 proxy_string = None 431 if self.proxy not in [None, '_none_']: 432 proxy_string = '%s' % self.proxy 433 if self.proxy_username is not None: 434 proxy_parsed = urlparse.urlsplit(self.proxy, allow_fragments=0) 435 proxy_proto = proxy_parsed[0] 436 proxy_host = proxy_parsed[1] 437 # http://foo:123 == ('http', 'foo:123', '', '', '') 438 # don't turn that into: http://foo:123? - bug#328121 439 if proxy_parsed[2] == '': 440 proxy_rest = '' 441 else: 442 proxy_rest = proxy_parsed[2] + '?' + proxy_parsed[3] 443 proxy_string = '%s://%s@%s%s' % (proxy_proto, 444 self.proxy_username, proxy_host, proxy_rest) 445 446 if self.proxy_password is not None: 447 proxy_string = '%s://%s:%s@%s%s' % (proxy_proto, 448 self.proxy_username, self.proxy_password, 449 proxy_host, proxy_rest) 450 451 if proxy_string is not None: 452 self._proxy_dict['http'] = proxy_string 453 self._proxy_dict['https'] = proxy_string 454 self._proxy_dict['ftp'] = proxy_string

455

456 - def __headersListFromDict(self, cache=True):

457 """Convert our dict of headers to a list of 2-tuples for urlgrabber.""" 458 headers = [] 459 460 for key in self.http_headers: 461 headers.append((key, self.http_headers[key])) 462 if not (cache or 'Pragma' in self.http_headers): 463 headers.append(('Pragma', 'no-cache')) 464 465 return headers

466

467 - def setupGrab(self):

468 warnings.warn('setupGrab() will go away in a future version of Yum.\n', 469 Errors.YumFutureDeprecationWarning, stacklevel=2) 470 self._setupGrab()

471

472 - def _setupGrab(self):

473 """sets up the grabber functions with the already stocked in urls for 474 the mirror groups""" 475 476 if self.failovermethod == 'roundrobin': 477 mgclass = urlgrabber.mirror.MGRandomOrder 478 else: 479 mgclass = urlgrabber.mirror.MirrorGroup 480 481 ugopts = self._default_grabopts() 482 self._grabfunc = URLGrabber(progress_obj=self.callback, 483 failure_callback=self.failure_obj, 484 interrupt_callback=self.interrupt_callback, 485 copy_local=self.copy_local, 486 reget='simple', 487 **ugopts) 488 489 self._grab = mgclass(self._grabfunc, self.urls, 490 failure_callback=self.mirror_failure_obj)

491

492 - def _default_grabopts(self, cache=True):

493 opts = { 'keepalive': self.keepalive, 494 'bandwidth': self.bandwidth, 495 'retry': self.retries, 496 'throttle': self.throttle, 497 'proxies': self.proxy_dict, 498 'timeout': self.timeout, 499 'http_headers': tuple(self.__headersListFromDict(cache=cache)), 500 'ssl_verify_peer': self.sslverify, 501 'ssl_verify_host': self.sslverify, 502 'ssl_ca_cert': self.sslcacert, 503 'ssl_cert': self.sslclientcert, 504 'ssl_key': self.sslclientkey, 505 'user_agent': default_grabber.opts.user_agent, 506 } 507 return opts

508

509 - def _getgrabfunc(self):

510 if not self._grabfunc or self._callbacks_changed: 511 self._setupGrab() 512 self._callbacks_changed = False 513 return self._grabfunc

514

515 - def _getgrab(self):

516 if not self._grab or self._callbacks_changed: 517 self._setupGrab() 518 self._callbacks_changed = False 519 return self._grab

520 521 grabfunc = property(lambda self: self._getgrabfunc()) 522 grab = property(lambda self: self._getgrab()) 523

524 - def _dirSetupMkdir_p(self, dpath):

525 """make the necessary directory path, if possible, raise on failure""" 526 if os.path.exists(dpath) and os.path.isdir(dpath): 527 return 528 529 if self.cache: 530 raise Errors.RepoError, "Cannot access repository dir %s" % dpath 531 532 try: 533 os.makedirs(dpath, mode=0755) 534 except OSError, e: 535 msg = "%s: %s %s: %s" % ("Error making cache directory", 536 dpath, "error was", e) 537 raise Errors.RepoError, msg

538

539 - def dirSetup(self):

540 """make the necessary dirs, if possible, raise on failure""" 541 542 cachedir = os.path.join(self.basecachedir, self.id) 543 pkgdir = os.path.join(cachedir, 'packages') 544 hdrdir = os.path.join(cachedir, 'headers') 545 self.setAttribute('_dir_setup_cachedir', cachedir) 546 self.setAttribute('_dir_setup_pkgdir', pkgdir) 547 self.setAttribute('_dir_setup_hdrdir', hdrdir) 548 self.setAttribute('_dir_setup_gpgdir', self.cachedir + '/gpgdir') 549 550 cookie = self.cachedir + '/' + self.metadata_cookie_fn 551 self.setAttribute('_dir_setup_metadata_cookie', cookie) 552 553 for dir in [self.cachedir, self.pkgdir]: 554 self._dirSetupMkdir_p(dir) 555 556 # if we're using a cachedir that's not the system one, copy over these 557 # basic items from the system one 558 self._preload_md_from_system_cache('repomd.xml') 559 self._preload_md_from_system_cache('cachecookie') 560 self._preload_md_from_system_cache('mirrorlist.txt') 561 self._preload_md_from_system_cache('metalink.xml')

562

563 - def _dirGetAttr(self, attr):

564 """ Make the directory attributes call .dirSetup() if needed. """ 565 attr = '_dir_setup_' + attr 566 if not hasattr(self, attr): 567 self.dirSetup() 568 return getattr(self, attr)

569 - def _dirSetAttr(self, attr, val):

570 """ Make the directory attributes call .dirSetup() if needed. """ 571 attr = '_dir_setup_' + attr 572 if not hasattr(self, attr): 573 self.dirSetup() 574 575 if attr == '_dir_setup_pkgdir': 576 if not hasattr(self, '_old_pkgdirs'): 577 self._old_pkgdirs = [] 578 self._old_pkgdirs.append(getattr(self, attr)) 579 580 ret = setattr(self, attr, val) 581 if attr in ('_dir_setup_pkgdir', ): 582 self._dirSetupMkdir_p(val) 583 return ret

584 cachedir = property(lambda self: self._dirGetAttr('cachedir')) 585 pkgdir = property(lambda self: self._dirGetAttr('pkgdir'), 586 lambda self, x: self._dirSetAttr('pkgdir', x)) 587 hdrdir = property(lambda self: self._dirGetAttr('hdrdir'), 588 lambda self, x: self._dirSetAttr('hdrdir', x)) 589 gpgdir = property(lambda self: self._dirGetAttr('gpgdir'), 590 lambda self, x: self._dirSetAttr('gpgdir', x)) 591 metadata_cookie = property(lambda self: self._dirGetAttr('metadata_cookie')) 592

593 - def baseurlSetup(self):

594 warnings.warn('baseurlSetup() will go away in a future version of Yum.\n', 595 Errors.YumFutureDeprecationWarning, stacklevel=2) 596 self._baseurlSetup()

597

598 - def _hack_mirrorlist_for_anaconda(self):

599 # Anaconda doesn't like having mirrorlist and metalink, so we allow 600 # mirrorlist to act like metalink. Except we'd really like to know which 601 # we have without parsing it ... and want to store it in the right 602 # place etc. 603 # So here is #1 hack: see if the metalin kis unset and the mirrorlist 604 # URL contains the string "metalink", if it does we copy it over. 605 if self.metalink: 606 return 607 if not self.mirrorlist: 608 return 609 if self.mirrorlist.find("metalink") == -1: 610 return 611 self.metalink = self.mirrorlist

612

613 - def _baseurlSetup(self):

614 """go through the baseurls and mirrorlists and populate self.urls 615 with valid ones, run self.check() at the end to make sure it worked""" 616 617 self.baseurl = self._replace_and_check_url(self.baseurl) 618 # FIXME: We put all the mirrors in .baseurl as well as 619 # .urls for backward compat. (see bottom of func). So we'll save this 620 # out for repolist -v ... or anything else wants to know the baseurl 621 self._orig_baseurl = self.baseurl 622 623 mirrorurls = [] 624 self._hack_mirrorlist_for_anaconda() 625 if self.metalink and not self.mirrorlistparsed: 626 # FIXME: This is kind of lying to API callers 627 mirrorurls.extend(list(self.metalink_data.urls())) 628 self.mirrorlistparsed = True 629 if self.mirrorlist and not self.mirrorlistparsed: 630 mirrorurls.extend(self._getMirrorList()) 631 self.mirrorlistparsed = True 632 633 self.mirrorurls = self._replace_and_check_url(mirrorurls) 634 self._urls = self.baseurl + self.mirrorurls 635 # if our mirrorlist is just screwed then make sure we unlink a mirrorlist cache 636 if len(self._urls) < 1: 637 if hasattr(self, 'mirrorlist_file') and os.path.exists(self.mirrorlist_file): 638 if not self.cache: 639 try: 640 misc.unlink_f(self.mirrorlist_file) 641 except (IOError, OSError), e: 642 print 'Could not delete bad mirrorlist file: %s - %s' % (self.mirrorlist_file, e) 643 else: 644 print 'removing mirrorlist with no valid mirrors: %s' % self.mirrorlist_file 645 # store them all back in baseurl for compat purposes 646 self.baseurl = self._urls 647 self.check()

648

649 - def _replace_and_check_url(self, url_list):

650 goodurls = [] 651 skipped = None 652 for url in url_list: 653 # obvious bogons get ignored b/c, we could get more interesting checks but <shrug> 654 if url in ['', None]: 655 continue 656 url = parser.varReplace(url, self.yumvar) 657 if url[-1] != '/': 658 url= url + '/' 659 (s,b,p,q,f,o) = urlparse.urlparse(url) 660 if s not in ['http', 'ftp', 'file', 'https']: 661 skipped = url 662 continue 663 else: 664 goodurls.append(url) 665 666 if skipped is not None: 667 # Caller cleans up for us. 668 if goodurls: 669 print 'YumRepo Warning: Some mirror URLs are not using ftp, http[s] or file.\n Eg. %s' % misc.to_utf8(skipped) 670 else: # And raises in this case 671 print 'YumRepo Error: All mirror URLs are not using ftp, http[s] or file.\n Eg. %s' % misc.to_utf8(skipped) 672 return goodurls

673

674 - def _geturls(self):

675 if not self._urls: 676 self._baseurlSetup() 677 return self._urls

678 679 urls = property(fget=lambda self: self._geturls(), 680 fset=lambda self, value: setattr(self, "_urls", value), 681 fdel=lambda self: setattr(self, "_urls", None)) 682

683 - def _getMetalink(self):

684 if not self._metalink: 685 self.metalink_filename = self.cachedir + '/' + 'metalink.xml' 686 local = self.metalink_filename + '.tmp' 687 if not self._metalinkCurrent(): 688 url = misc.to_utf8(self.metalink) 689 ugopts = self._default_grabopts() 690 try: 691 ug = URLGrabber(progress_obj = self.callback, **ugopts) 692 result = ug.urlgrab(url, local, text=self.id + "/metalink") 693 694 except urlgrabber.grabber.URLGrabError, e: 695 if not os.path.exists(self.metalink_filename): 696 msg = ("Cannot retrieve metalink for repository: %s. " 697 "Please verify its path and try again" % self ) 698 raise Errors.RepoError, msg 699 # Now, we have an old usable metalink, so we can't move to 700 # a newer repomd.xml ... or checksums won't match. 701 print "Could not get metalink %s error was\n%s: %s" % (url, e.args[0], misc.to_unicode(e.args[1])) 702 self._metadataCurrent = True 703 704 if not self._metadataCurrent: 705 try: 706 self._metalink = metalink.MetaLinkRepoMD(result) 707 shutil.move(result, self.metalink_filename) 708 except metalink.MetaLinkRepoErrorParseFail, e: 709 # Downloaded file failed to parse, revert (dito. above): 710 print "Could not parse metalink %s error was \n%s"%(url, e) 711 self._metadataCurrent = True 712 misc.unlink_f(result) 713 714 if self._metadataCurrent: 715 self._metalink = metalink.MetaLinkRepoMD(self.metalink_filename) 716 717 return self._metalink

718 719 metalink_data = property(fget=lambda self: self._getMetalink(), 720 fset=lambda self, value: setattr(self, "_metalink", 721 value), 722 fdel=lambda self: setattr(self, "_metalink", None)) 723

724 - def _getFile(self, url=None, relative=None, local=None, start=None, end=None, 725 copy_local=None, checkfunc=None, text=None, reget='simple', 726 cache=True, size=None):

727 """retrieve file from the mirrorgroup for the repo 728 relative to local, optionally get range from 729 start to end, also optionally retrieve from a specific baseurl""" 730 731 # if local or relative is None: raise an exception b/c that shouldn't happen 732 # if url is not None - then do a grab from the complete url - not through 733 # the mirror, raise errors as need be 734 # if url is None do a grab via the mirror group/grab for the repo 735 # return the path to the local file 736 737 # if copylocal isn't specified pickup the repo-defined attr 738 if copy_local is None: 739 copy_local = self.copy_local 740 741 if local is None or relative is None: 742 raise Errors.RepoError, \ 743 "get request for Repo %s, gave no source or dest" % self.id 744 745 if self.cache == 1: 746 if os.path.exists(local): # FIXME - we should figure out a way 747 return local # to run the checkfunc from here 748 749 else: # ain't there - raise 750 raise Errors.RepoError, \ 751 "Caching enabled but no local cache of %s from %s" % (local, 752 753 self) 754 755 if url: 756 (scheme, netloc, path, query, fragid) = urlparse.urlsplit(url) 757 758 if self.mediaid and self.mediafunc: 759 discnum = 1 760 if url: 761 if scheme == "media" and fragid: 762 discnum = int(fragid) 763 try: 764 # FIXME: we need to figure out what really matters to 765 # pass to the media grabber function here 766 result = self.mediafunc(local = local, checkfunc = checkfunc, relative = relative, text = text, copy_local = copy_local, url = url, mediaid = self.mediaid, name = self.name, discnum = discnum, range = (start, end)) 767 return result 768 except Errors.MediaError, e: 769 verbose_logger.log(logginglevels.DEBUG_2, "Error getting package from media; falling back to url %s" %(e,)) 770 771 if url and scheme != "media": 772 ugopts = self._default_grabopts(cache=cache) 773 ug = URLGrabber(progress_obj = self.callback, 774 copy_local = copy_local, 775 reget = reget, 776 failure_callback = self.failure_obj, 777 interrupt_callback=self.interrupt_callback, 778 checkfunc=checkfunc, 779 size=size, 780 **ugopts) 781 782 remote = url + '/' + relative 783 784 try: 785 result = ug.urlgrab(misc.to_utf8(remote), local, 786 text=misc.to_utf8(text), 787 range=(start, end), 788 ) 789 except URLGrabError, e: 790 errstr = "failed to retrieve %s from %s\nerror was %s" % (relative, self.id, e) 791 if self.mirrorurls: 792 errstr +="\n You could try running: yum clean expire-cache" 793 errstr +="\n To get a new set of mirrors." 794 if e.errno == 256: 795 raise Errors.NoMoreMirrorsRepoError, errstr 796 else: 797 raise Errors.RepoError, errstr 798 799 800 else: 801 headers = tuple(self.__headersListFromDict(cache=cache)) 802 try: 803 result = self.grab.urlgrab(misc.to_utf8(relative), local, 804 text = misc.to_utf8(text), 805 range = (start, end), 806 copy_local=copy_local, 807 reget = reget, 808 checkfunc=checkfunc, 809 http_headers=headers, 810 size=size 811 ) 812 except URLGrabError, e: 813 errstr = "failure: %s from %s: %s" % (relative, self.id, e) 814 if e.errno == 256: 815 raise Errors.NoMoreMirrorsRepoError, errstr 816 else: 817 raise Errors.RepoError, errstr 818 819 return result

820 __get = _getFile 821

822 - def getPackage(self, package, checkfunc=None, text=None, cache=True):

823 remote = package.relativepath 824 local = package.localPkg() 825 basepath = package.basepath 826 827 if self._preload_pkg_from_system_cache(package): 828 if package.verifyLocalPkg(): 829 return local 830 misc.unlink_f(local) 831 832 return self._getFile(url=basepath, 833 relative=remote, 834 local=local, 835 checkfunc=checkfunc, 836 text=text, 837 cache=cache, 838 size=package.size, 839 )

840

841 - def getHeader(self, package, checkfunc = None, reget = 'simple', 842 cache = True):

843 844 remote = package.relativepath 845 local = package.localHdr() 846 start = package.hdrstart 847 end = package.hdrend 848 size = end-start 849 basepath = package.basepath 850 # yes, I know, don't ask 851 if not os.path.exists(self.hdrdir): 852 os.makedirs(self.hdrdir) 853 854 return self._getFile(url=basepath, relative=remote, local=local, start=start, 855 reget=None, end=end, checkfunc=checkfunc, copy_local=1, 856 cache=cache, size=size, 857 )

858

859 - def metadataCurrent(self):

860 """Check if there is a metadata_cookie and check its age. If the 861 age of the cookie is less than metadata_expire time then return true 862 else return False. This result is cached, so that metalink/repomd.xml 863 are synchronized.""" 864 if self._metadataCurrent is not None: 865 return self._metadataCurrent 866 867 mC_def = self.withinCacheAge(self.metadata_cookie, self.metadata_expire) 868 if not mC_def: # Normal path... 869 self._metadataCurrent = mC_def 870 return mC_def 871 872 # Edge cases, both repomd.xml and metalink (if used). Must exist. 873 repomdfn = self.cachedir + '/' + 'repomd.xml' 874 if not os.path.exists(repomdfn): 875 self._metadataCurrent = False 876 return False 877 878 self._hack_mirrorlist_for_anaconda() 879 mlfn = self.cachedir + '/' + 'metalink.xml' 880 if self.metalink and not os.path.exists(mlfn): 881 self._metadataCurrent = False 882 return False 883 884 self._metadataCurrent = True 885 return True

886 887 # The metalink _shouldn't_ be newer than the repomd.xml or the checksums 888 # will be off, but we only really care when we are downloading the 889 # repomd.xml ... so keep it in mind that they can be off on disk. 890 # Also see _getMetalink()

891 - def _metalinkCurrent(self):

892 if self._metadataCurrent is not None: 893 return self._metadataCurrent 894 895 if self.cache and not os.path.exists(self.metalink_filename): 896 raise Errors.RepoError, 'Cannot find metalink.xml file for %s' %self 897 898 if self.cache: 899 self._metadataCurrent = True 900 elif not os.path.exists(self.metalink_filename): 901 self._metadataCurrent = False 902 elif self.withinCacheAge(self.metadata_cookie, self.metadata_expire): 903 self._metadataCurrent = True 904 else: 905 self._metadataCurrent = False 906 return self._metadataCurrent

907

908 - def withinCacheAge(self, myfile, expiration_time):

909 """check if any file is older than a certain amount of time. Used for 910 the cachecookie and the mirrorlist 911 return True if w/i the expiration time limit 912 false if the time limit has expired 913 914 Additionally compare the file to age of the newest .repo or yum.conf 915 file. If any of them are newer then invalidate the cache 916 """ 917 918 # -1 is special and should never get refreshed 919 if expiration_time == -1 and os.path.exists(myfile): 920 return True 921 val = False 922 if os.path.exists(myfile): 923 cookie_info = os.stat(myfile) 924 if cookie_info[8] + expiration_time > time.time(): 925 val = True 926 # WE ARE FROM THE FUTURE!!!! 927 elif cookie_info[8] > time.time(): 928 val = False 929 930 # make sure none of our config files for this repo are newer than 931 # us 932 if cookie_info[8] < int(self.repo_config_age): 933 val = False 934 935 return val

936

937 - def setMetadataCookie(self):

938 """if possible, set touch the metadata_cookie file""" 939 940 check = self.metadata_cookie 941 if not os.path.exists(self.metadata_cookie): 942 check = self.cachedir 943 944 if os.access(check, os.W_OK): 945 fo = open(self.metadata_cookie, 'w+') 946 fo.close() 947 del fo

948

949 - def setup(self, cache, mediafunc = None, gpg_import_func=None, confirm_func=None):

950 try: 951 self.cache = cache 952 self.mediafunc = mediafunc 953 self.gpg_import_func = gpg_import_func 954 self.confirm_func = confirm_func 955 except Errors.RepoError, e: 956 raise 957 if not self.mediafunc and self.mediaid and not self.mirrorlist and not self.baseurl: 958 verbose_logger.log(logginglevels.DEBUG_2, "Disabling media repo for non-media-aware frontend") 959 self.enabled = False

960

961 - def _cachingRepoXML(self, local):

962 """ Should we cache the current repomd.xml """ 963 if self.cache and not os.path.exists(local): 964 raise Errors.RepoError, 'Cannot find repomd.xml file for %s' % self 965 if self.cache or self.metadataCurrent(): 966 return True 967 return False

968

969 - def _getFileRepoXML(self, local, text=None, grab_can_fail=None):

970 """ Call _getFile() for the repomd.xml file. """ 971 checkfunc = (self._checkRepoXML, (), {}) 972 if grab_can_fail is None: 973 grab_can_fail = 'old_repo_XML' in self._oldRepoMDData 974 tfname = '' 975 try: 976 # This is named so that "yum clean metadata" picks it up 977 tfname = tempfile.mktemp(prefix='repomd', suffix="tmp.xml", 978 dir=os.path.dirname(local)) 979 result = self._getFile(relative=self.repoMDFile, 980 local=tfname, 981 copy_local=1, 982 text=text, 983 reget=None, 984 checkfunc=checkfunc, 985 cache=self.http_caching == 'all', 986 size=102400) # setting max size as 100K 987 988 except URLGrabError, e: 989 misc.unlink_f(tfname) 990 if grab_can_fail: 991 return None 992 raise Errors.RepoError, 'Error downloading file %s: %s' % (local, e) 993 except (Errors.NoMoreMirrorsRepoError, Errors.RepoError): 994 misc.unlink_f(tfname) 995 if grab_can_fail: 996 return None 997 raise 998 999 # This should always work... 1000 try: 1001 os.rename(result, local) 1002 except: 1003 # But in case it doesn't... 1004 misc.unlink_f(tfname) 1005 if grab_can_fail: 1006 return None 1007 raise Errors.RepoError, 'Error renaming file %s to %s' % (result, 1008 local) 1009 return local

1010

1011 - def _parseRepoXML(self, local, parse_can_fail=None):

1012 """ Parse the repomd.xml file. """ 1013 try: 1014 return repoMDObject.RepoMD(self.id, local) 1015 except Errors.RepoMDError, e: 1016 if parse_can_fail is None: 1017 parse_can_fail = 'old_repo_XML' in self._oldRepoMDData 1018 if parse_can_fail: 1019 return None 1020 raise Errors.RepoError, 'Error importing repomd.xml from %s: %s' % (self, e)

1021

1022 - def _saveOldRepoXML(self, local):

1023 """ If we have an older repomd.xml file available, save it out. """ 1024 # Cleanup old trash... 1025 for fname in glob.glob(self.cachedir + "/*.old.tmp"): 1026 misc.unlink_f(fname) 1027 1028 if os.path.exists(local): 1029 old_local = local + '.old.tmp' # locked, so this is ok 1030 shutil.copy2(local, old_local) 1031 xml = self._parseRepoXML(old_local, True) 1032 if xml is None: 1033 return None 1034 self._oldRepoMDData = {'old_repo_XML' : xml, 'local' : local, 1035 'old_local' : old_local, 'new_MD_files' : []} 1036 return xml 1037 return None

1038

1039 - def _revertOldRepoXML(self):

1040 """ If we have older data available, revert to it. """ 1041 if 'old_repo_XML' not in self._oldRepoMDData: 1042 self._oldRepoMDData = {} 1043 return 1044 1045 # Unique names mean the rename doesn't work anymore. 1046 for fname in self._oldRepoMDData['new_MD_files']: 1047 misc.unlink_f(fname) 1048 1049 old_data = self._oldRepoMDData 1050 self._oldRepoMDData = {} 1051 1052 if 'old_local' in old_data: 1053 os.rename(old_data['old_local'], old_data['local']) 1054 1055 self._repoXML = old_data['old_repo_XML'] 1056 1057 if 'old_MD_files' not in old_data: 1058 return 1059 for revert in old_data['old_MD_files']: 1060 os.rename(revert + '.old.tmp', revert)

1061

1062 - def _doneOldRepoXML(self):

1063 """ Done with old data, delete it. """ 1064 old_data = self._oldRepoMDData 1065 self._oldRepoMDData = {} 1066 1067 if 'old_local' in old_data: 1068 misc.unlink_f(old_data['old_local']) 1069 1070 if 'old_MD_files' not in old_data: 1071 return 1072 for revert in old_data['old_MD_files']: 1073 misc.unlink_f(revert + '.old.tmp')

1074

1075 - def _get_mdtype_data(self, mdtype, repoXML=None):

1076 if repoXML is None: 1077 repoXML = self.repoXML 1078 1079 if mdtype == 'group' and 'group_gz' in repoXML.fileTypes(): 1080 mdtype = 'group_gz' 1081 if (mdtype in ['other', 'filelists', 'primary'] and 1082 self._check_db_version(mdtype + '_db', repoXML=repoXML)): 1083 mdtype += '_db' 1084 1085 if repoXML.repoData.has_key(mdtype): 1086 return (mdtype, repoXML.getData(mdtype)) 1087 return (mdtype, None)

1088

1089 - def _get_mdtype_fname(self, data, compressed=False):

1090 (r_base, remote) = data.location 1091 local = self.cachedir + '/' + os.path.basename(remote) 1092 1093 if compressed: # DB file, we need the uncompressed version 1094 local = local.replace('.bz2', '') 1095 return local

1096

1097 - def _groupCheckDataMDNewer(self):

1098 """ We check the timestamps, if any of the timestamps for the 1099 "new" data is older than what we have ... we revert. """ 1100 1101 if 'old_repo_XML' not in self._oldRepoMDData: 1102 return True 1103 old_repo_XML = self._oldRepoMDData['old_repo_XML'] 1104 1105 if (self.timestamp_check and 1106 old_repo_XML.timestamp > self.repoXML.timestamp): 1107 logger.warning("Not using downloaded repomd.xml because it is " 1108 "older than what we have:\n" 1109 " Current : %s\n Downloaded: %s" % 1110 (time.ctime(old_repo_XML.timestamp), 1111 time.ctime(self.repoXML.timestamp))) 1112 return False 1113 return True

1114 1115 @staticmethod

1116 - def _checkRepoXMLMetalink(repoXML, repomd):

1117 """ Check parsed repomd.xml against metalink.repomd data. """ 1118 if repoXML.timestamp != repomd.timestamp: 1119 return False 1120 if repoXML.length != repomd.size: 1121 return False 1122 1123 for checksum in repoXML.checksums: 1124 if checksum not in repomd.chksums: 1125 continue 1126 1127 if repoXML.checksums[checksum] != repomd.chksums[checksum]: 1128 return False 1129 1130 # If we don't trust the checksum, then don't generate it in 1131 # repoMDObject(). 1132 return True 1133 1134 return False

1135

1136 - def _checkRepoMetalink(self, repoXML=None, metalink_data=None):

1137 """ Check the repomd.xml against the metalink data, if we have it. """ 1138 1139 if repoXML is None: 1140 repoXML = self._repoXML 1141 if metalink_data is None: 1142 metalink_data = self.metalink_data 1143 1144 if self._checkRepoXMLMetalink(repoXML, metalink_data.repomd): 1145 return True 1146 1147 # FIXME: We probably want to skip to the first mirror which has the 1148 # latest repomd.xml, but say "if we can't find one, use the newest old 1149 # repomd.xml" ... alas. that's not so easy to do in urlgrabber atm. 1150 for repomd in self.metalink_data.old_repomds: 1151 if self._checkRepoXMLMetalink(repoXML, repomd): 1152 verbose_logger.log(logginglevels.DEBUG_2, 1153 "Using older repomd.xml\n" 1154 " Latest: %s\n" 1155 " Using: %s" % 1156 (time.ctime(metalink_data.repomd.timestamp), 1157 time.ctime(repomd.timestamp))) 1158 return True 1159 return False

1160

1161 - def _latestRepoXML(self, local):

1162 """ Save the Old Repo XML, and if it exists check to see if it's the 1163 latest available given the metalink data. """ 1164 1165 oxml = self._saveOldRepoXML(local) 1166 if not oxml: # No old repomd.xml data 1167 return False 1168 1169 self._hack_mirrorlist_for_anaconda() 1170 if not self.metalink: # Nothing to check it against 1171 return False 1172 1173 # Get the latest metalink, and the latest repomd data from it 1174 repomd = self.metalink_data.repomd 1175 1176 if self.timestamp_check and oxml.timestamp > repomd.timestamp: 1177 # We have something "newer" than the latest, and have timestamp 1178 # checking which will kill anything passing the metalink check. 1179 return True 1180 1181 # Do we have the latest repomd already 1182 return self._checkRepoXMLMetalink(oxml, repomd)

1183

1184 - def _commonLoadRepoXML(self, text, mdtypes=None):

1185 """ Common LoadRepoXML for instant and group, returns False if you 1186 should just return. """ 1187 local = self.cachedir + '/repomd.xml' 1188 if self._repoXML is not None: 1189 return False 1190 1191 if self._cachingRepoXML(local): 1192 caching = True 1193 result = local 1194 else: 1195 caching = False 1196 if self._latestRepoXML(local): 1197 result = local 1198 old_data = self._oldRepoMDData 1199 self._repoXML = old_data['old_repo_XML'] 1200 else: 1201 result = self._getFileRepoXML(local, text) 1202 if result is None: 1203 # Ignore this as we have a copy 1204 self._revertOldRepoXML() 1205 return False 1206 1207 # if we have a 'fresh' repomd.xml then update the cookie 1208 self.setMetadataCookie() 1209 1210 if self._repoXML is None: 1211 self._repoXML = self._parseRepoXML(result) 1212 if self._repoXML is None: 1213 self._revertOldRepoXML() 1214 return False 1215 1216 self._using_old_MD = caching 1217 if caching: 1218 return False # Skip any work. 1219 1220 if not self._groupCheckDataMDNewer(): 1221 self._revertOldRepoXML() 1222 return False 1223 return True

1224

1225 - def _check_db_version(self, mdtype, repoXML=None):

1226 if repoXML is None: 1227 repoXML = self.repoXML 1228 if mdtype in repoXML.repoData: 1229 if DBVERSION == repoXML.repoData[mdtype].dbversion: 1230 return True 1231 return False

1232 1233 # mmdtype is unused, but in theory was == primary 1234 # dbmtype == primary_db etc.

1235 - def _groupCheckDataMDValid(self, data, dbmdtype, mmdtype, file_check=False):

1236 """ Check that we already have this data, and that it's valid. Given 1237 the DB mdtype and the main mdtype (no _db suffix). """ 1238 1239 if data is None: 1240 return None 1241 1242 if not file_check: 1243 compressed = dbmdtype.endswith("_db") 1244 local = self._get_mdtype_fname(data, compressed) 1245 else: 1246 compressed = False 1247 local = self._get_mdtype_fname(data, False) 1248 if not os.path.exists(local): 1249 local = local.replace('.bz2', '') 1250 compressed = True 1251 # If we can, make a copy of the system-wide-cache version of this file, 1252 # note that we often don't get here. So we also do this in 1253 # YumPackageSack.populate ... and we look for the uncompressed versions 1254 # in retrieveMD. 1255 self._preload_md_from_system_cache(os.path.basename(local)) 1256 if not self._checkMD(local, dbmdtype, openchecksum=compressed, 1257 data=data, check_can_fail=True): 1258 return None 1259 1260 return local

1261

1262 - def _commonRetrieveDataMD(self, mdtypes=None):

1263 """ Retrieve any listed mdtypes, and revert if there was a failure. 1264 Also put any of the non-valid mdtype files from the old_repo_XML 1265 into the delete list, this means metadata can change filename 1266 without us leaking it. """ 1267 1268 def _mdtype_eq(omdtype, odata, nmdtype, ndata): 1269 """ Check if two returns from _get_mdtype_data() are equal. """ 1270 if ndata is None: 1271 return False 1272 if omdtype != nmdtype: 1273 return False 1274 if odata.checksum != ndata.checksum: 1275 return False 1276 # If we turn --unique-md-filenames on without chaning the data, 1277 # then we'll get different filenames, but the same checksum. 1278 # Atm. just say they are different, to make sure we delete the 1279 # old files. 1280 orname = os.path.basename(odata.location[1]) 1281 nrname = os.path.basename(ndata.location[1]) 1282 if orname != nrname: 1283 return False 1284 return True

1285 1286 all_mdtypes = self.retrieved.keys() 1287 if mdtypes is None: 1288 mdtypes = all_mdtypes 1289 1290 reverts = [] 1291 if 'old_repo_XML' not in self._oldRepoMDData: 1292 old_repo_XML = None 1293 else: 1294 old_repo_XML = self._oldRepoMDData['old_repo_XML'] 1295 self._oldRepoMDData['old_MD_files'] = reverts 1296 1297 # Inited twice atm. ... sue me 1298 self._oldRepoMDData['new_MD_files'] = [] 1299 downloading_with_size = [] 1300 downloading_no_size = [] 1301 for mdtype in all_mdtypes: 1302 (nmdtype, ndata) = self._get_mdtype_data(mdtype) 1303 1304 if old_repo_XML: 1305 (omdtype, odata) = self._get_mdtype_data(mdtype, 1306 repoXML=old_repo_XML) 1307 local = self._groupCheckDataMDValid(odata, omdtype,mdtype,True) 1308 if local: 1309 if _mdtype_eq(omdtype, odata, nmdtype, ndata): 1310 continue # If they are the same do nothing 1311 1312 # Move this version, we _may_ get a new one. 1313 # We delete it on success, revert it back on failure. 1314 # We don't copy as we know it's bad due to above test. 1315 os.rename(local, local + '.old.tmp') 1316 reverts.append(local) 1317 1318 if ndata is None: # Doesn't exist in this repo 1319 continue 1320 1321 if mdtype not in mdtypes: 1322 continue 1323 1324 # No old repomd data, but we might still have uncompressed MD 1325 if self._groupCheckDataMDValid(ndata, nmdtype, mdtype): 1326 continue 1327 1328 if ndata.size is None: 1329 downloading_no_size.append((ndata, nmdtype)) 1330 else: 1331 downloading_with_size.append((ndata, nmdtype)) 1332 1333 if len(downloading_with_size) == 1: 1334 downloading_no_size.extend(downloading_with_size) 1335 downloading_with_size = [] 1336 1337 remote_size = 0 1338 local_size = 0 1339 for (ndata, nmdtype) in downloading_with_size: # Get total size... 1340 remote_size += int(ndata.size) 1341 1342 for (ndata, nmdtype) in downloading_with_size: 1343 urlgrabber.progress.text_meter_total_size(remote_size, local_size) 1344 if not self._retrieveMD(nmdtype, retrieve_can_fail=True): 1345 self._revertOldRepoXML() 1346 return False 1347 local_size += int(ndata.size) 1348 urlgrabber.progress.text_meter_total_size(0) 1349 for (ndata, nmdtype) in downloading_no_size: 1350 if not self._retrieveMD(nmdtype, retrieve_can_fail=True): 1351 self._revertOldRepoXML() 1352 return False 1353 1354 for (ndata, nmdtype) in downloading_with_size + downloading_no_size: 1355 local = self._get_mdtype_fname(ndata, False) 1356 if nmdtype.endswith("_db"): # Uncompress any .sqlite.bz2 files 1357 dl_local = local 1358 local = local.replace('.bz2', '') 1359 misc.bunzipFile(dl_local, local) 1360 misc.unlink_f(dl_local) 1361 self._oldRepoMDData['new_MD_files'].append(local) 1362 1363 self._doneOldRepoXML() 1364 return True

1365

1366 - def _groupLoadRepoXML(self, text=None, mdtypes=None):

1367 """ Retrieve the new repomd.xml from the repository, then check it 1368 and parse it. If it fails we revert to the old version and pretend 1369 that is fine. If the new repomd.xml requires new version of files 1370 that we have, like updateinfo.xml, we download those too and if any 1371 of those fail, we again revert everything and pretend old data is 1372 good. """ 1373 1374 if self._commonLoadRepoXML(text): 1375 self._commonRetrieveDataMD(mdtypes)

1376

1377 - def _mdpolicy2mdtypes(self):

1378 md_groups = {'instant' : [], 1379 'group:primary' : ['primary'], 1380 'group:small' : ["primary", "updateinfo"], 1381 'group:main' : ["primary", "group", "filelists", 1382 "updateinfo", "prestodelta"]} 1383 mdtypes = set() 1384 if type(self.mdpolicy) in types.StringTypes: 1385 mdtypes.update(md_groups.get(self.mdpolicy, [self.mdpolicy])) 1386 else: 1387 for mdpolicy in self.mdpolicy: 1388 mdtypes.update(md_groups.get(mdpolicy, [mdpolicy])) 1389 1390 if not mdtypes or 'group:all' in mdtypes: 1391 mdtypes = None 1392 else: 1393 mdtypes = sorted(list(mdtypes)) 1394 return mdtypes

1395

1396 - def _loadRepoXML(self, text=None):

1397 """retrieve/check/read in repomd.xml from the repository""" 1398 try: 1399 return self._groupLoadRepoXML(text, self._mdpolicy2mdtypes()) 1400 except KeyboardInterrupt: 1401 self._revertOldRepoXML() # Undo metadata cookie? 1402 raise 1403 raise Errors.RepoError, 'Bad loadRepoXML policy: %s' % (self.mdpolicy)

1404

1405 - def _getRepoXML(self):

1406 if self._repoXML: 1407 return self._repoXML 1408 try: 1409 self._loadRepoXML(text=self) 1410 except Errors.RepoError, e: 1411 msg = ("Cannot retrieve repository metadata (repomd.xml) for repository: %s. " 1412 "Please verify its path and try again" % self ) 1413 raise Errors.RepoError, msg 1414 return self._repoXML

1415 1416 1417 repoXML = property(fget=lambda self: self._getRepoXML(), 1418 fset=lambda self, val: setattr(self, "_repoXML", val), 1419 fdel=lambda self: setattr(self, "_repoXML", None)) 1420

1421 - def _checkRepoXML(self, fo):

1422 if type(fo) is types.InstanceType: 1423 filepath = fo.filename 1424 else: 1425 filepath = fo 1426 1427 if self.repo_gpgcheck: 1428 1429 if misc.gpgme is None: 1430 raise URLGrabError(-1, 'pygpgme is not working so repomd.xml can not be verified for %s' % (self)) 1431 1432 sigfile = self.cachedir + '/repomd.xml.asc' 1433 try: 1434 result = self._getFile(relative='repodata/repomd.xml.asc', 1435 copy_local=1, 1436 local = sigfile, 1437 text='%s/signature' % self.id, 1438 reget=None, 1439 checkfunc=None, 1440 cache=self.http_caching == 'all', 1441 size=102400) 1442 except URLGrabError, e: 1443 raise URLGrabError(-1, 'Error finding signature for repomd.xml for %s: %s' % (self, e)) 1444 1445 valid = misc.valid_detached_sig(result, filepath, self.gpgdir) 1446 if not valid and self.gpg_import_func: 1447 try: 1448 self.gpg_import_func(self, self.confirm_func) 1449 except Errors.YumBaseError, e: 1450 raise URLGrabError(-1, 'Gpg Keys not imported, cannot verify repomd.xml for repo %s' % (self)) 1451 valid = misc.valid_detached_sig(result, filepath, self.gpgdir) 1452 1453 if not valid: 1454 raise URLGrabError(-1, 'repomd.xml signature could not be verified for %s' % (self)) 1455 1456 try: 1457 repoXML = repoMDObject.RepoMD(self.id, filepath) 1458 except Errors.RepoMDError, e: 1459 raise URLGrabError(-1, 'Error importing repomd.xml for %s: %s' % (self, e)) 1460 1461 self._hack_mirrorlist_for_anaconda() 1462 if self.metalink and not self._checkRepoMetalink(repoXML): 1463 raise URLGrabError(-1, 'repomd.xml does not match metalink for %s' % 1464 self)

1465 1466

1467 - def checkMD(self, fn, mdtype, openchecksum=False):

1468 """check the metadata type against its checksum""" 1469 return self._checkMD(fn, mdtype, openchecksum)

1470

1471 - def _checkMD(self, fn, mdtype, openchecksum=False, 1472 data=None, check_can_fail=False):

1473 """ Internal function, use .checkMD() from outside yum. """ 1474 1475 thisdata = data # So the argument name is nicer 1476 if thisdata is None: 1477 thisdata = self.repoXML.getData(mdtype) 1478 1479 # Note openchecksum means do it after you've uncompressed the data. 1480 if openchecksum: 1481 (r_ctype, r_csum) = thisdata.openchecksum # get the remote checksum 1482 size = thisdata.opensize 1483 else: 1484 (r_ctype, r_csum) = thisdata.checksum # get the remote checksum 1485 size = thisdata.size 1486 1487 if type(fn) == types.InstanceType: # this is an urlgrabber check 1488 file = fn.filename 1489 else: 1490 file = fn 1491 1492 if size is not None: 1493 size = int(size) 1494 1495 try: # get the local checksum 1496 l_csum = self._checksum(r_ctype, file, datasize=size) 1497 except Errors.RepoError, e: 1498 if check_can_fail: 1499 return None 1500 raise URLGrabError(-3, 'Error performing checksum') 1501 1502 if l_csum == r_csum: 1503 return 1 1504 else: 1505 if check_can_fail: 1506 return None 1507 raise URLGrabError(-1, 'Metadata file does not match checksum')

1508 1509 1510

1511 - def retrieveMD(self, mdtype):

1512 """base function to retrieve metadata files from the remote url 1513 returns the path to the local metadata file of a 'mdtype' 1514 mdtype can be 'primary', 'filelists', 'other' or 'group'.""" 1515 return self._retrieveMD(mdtype)

1516

1517 - def _retrieveMD(self, mdtype, retrieve_can_fail=False):

1518 """ Internal function, use .retrieveMD() from outside yum. """ 1519 # Note that this can raise Errors.RepoMDError if mdtype doesn't exist 1520 # for this repo. 1521 thisdata = self.repoXML.getData(mdtype) 1522 1523 (r_base, remote) = thisdata.location 1524 fname = os.path.basename(remote) 1525 local = self.cachedir + '/' + fname 1526 1527 if self.retrieved.has_key(mdtype): 1528 if self.retrieved[mdtype]: # got it, move along 1529 return local 1530 1531 if self.cache == 1: 1532 if os.path.exists(local): 1533 try: 1534 self.checkMD(local, mdtype) 1535 except URLGrabError, e: 1536 raise Errors.RepoError, \ 1537 "Caching enabled and local cache: %s does not match checksum" % local 1538 else: 1539 return local 1540 1541 else: # ain't there - raise 1542 raise Errors.RepoError, \ 1543 "Caching enabled but no local cache of %s from %s" % (local, 1544 self) 1545 1546 if (os.path.exists(local) or 1547 self._preload_md_from_system_cache(os.path.basename(local))): 1548 if self._checkMD(local, mdtype, check_can_fail=True): 1549 self.retrieved[mdtype] = 1 1550 return local # it's the same return the local one 1551 1552 try: 1553 checkfunc = (self.checkMD, (mdtype,), {}) 1554 text = "%s/%s" % (self.id, mdtype) 1555 local = self._getFile(relative=remote, 1556 local=local, 1557 copy_local=1, 1558 checkfunc=checkfunc, 1559 text=text, 1560 cache=self.http_caching == 'all', 1561 size=thisdata.size) 1562 except (Errors.NoMoreMirrorsRepoError, Errors.RepoError): 1563 if retrieve_can_fail: 1564 return None 1565 raise 1566 except URLGrabError, e: 1567 if retrieve_can_fail: 1568 return None 1569 raise Errors.RepoError, \ 1570 "Could not retrieve %s matching remote checksum from %s" % (local, self) 1571 else: 1572 self.retrieved[mdtype] = 1 1573 return local

1574 1575

1576 - def getPrimaryXML(self):

1577 """this gets you the path to the primary.xml file, retrieving it if we 1578 need a new one""" 1579 1580 return self.retrieveMD('primary')

1581 1582

1583 - def getFileListsXML(self):

1584 """this gets you the path to the filelists.xml file, retrieving it if we 1585 need a new one""" 1586 1587 return self.retrieveMD('filelists')

1588

1589 - def getOtherXML(self):

1590 return self.retrieveMD('other')

1591

1592 - def getGroups(self):

1593 """gets groups and returns group file path for the repository, if there 1594 is none it returns None""" 1595 if 'group_gz' in self.repoXML.fileTypes(): 1596 return self._retrieveMD('group_gz', retrieve_can_fail=True) 1597 return self._retrieveMD('group', retrieve_can_fail=True)

1598

1599 - def setCallback(self, callback):

1600 self.callback = callback 1601 self._callbacks_changed = True

1602

1603 - def setFailureObj(self, failure_obj):

1604 self.failure_obj = failure_obj 1605 self._callbacks_changed = True

1606

1607 - def setMirrorFailureObj(self, failure_obj):

1608 self.mirror_failure_obj = failure_obj 1609 self._callbacks_changed = True

1610

1611 - def setInterruptCallback(self, callback):

1612 self.interrupt_callback = callback 1613 self._callbacks_changed = True

1614

1615 - def _readMirrorList(self, fo, url=None):

1616 """ read the mirror list from the specified file object """ 1617 returnlist = [] 1618 1619 content = [] 1620 if fo is not None: 1621 try: 1622 content = fo.readlines() 1623 except Exception, e: 1624 if url is None: # Shouldn't happen 1625 url = "<unknown>" 1626 print "Could not read mirrorlist %s, error was \n%s" %(url, e) 1627 content = [] 1628 for line in content: 1629 if re.match('\s*(#|$)', line): 1630 continue 1631 mirror = line.rstrip() # no more trailing \n's 1632 mirror = mirror.replace('$ARCH', '$BASEARCH') 1633 returnlist.append(mirror) 1634 1635 return (returnlist, content)

1636

1637 - def _getMirrorList(self):

1638 """retrieve an up2date-style mirrorlist file from our mirrorlist url, 1639 also save the file to the local repo dir and use that if cache expiry 1640 not expired 1641 1642 we also s/$ARCH/$BASEARCH/ and move along 1643 return the baseurls from the mirrorlist file 1644 """ 1645 self.mirrorlist_file = self.cachedir + '/' + 'mirrorlist.txt' 1646 fo = None 1647 1648 cacheok = False 1649 if self.withinCacheAge(self.mirrorlist_file, self.mirrorlist_expire): 1650 cacheok = True 1651 fo = open(self.mirrorlist_file, 'r') 1652 url = 'file://' + self.mirrorlist_file # just to keep self._readMirrorList(fo,url) happy 1653 else: 1654 url = self.mirrorlist 1655 scheme = urlparse.urlparse(url)[0] 1656 if scheme == '': 1657 url = 'file://' + url 1658 ugopts = self._default_grabopts() 1659 try: 1660 fo = urlgrabber.grabber.urlopen(url, **ugopts) 1661 except urlgrabber.grabber.URLGrabError, e: 1662 print "Could not retrieve mirrorlist %s error was\n%s: %s" % (url, e.args[0], misc.to_unicode(e.args[1])) 1663 fo = None 1664 1665 (returnlist, content) = self._readMirrorList(fo, url) 1666 1667 if returnlist: 1668 if not self.cache and not cacheok: 1669 output = open(self.mirrorlist_file, 'w') 1670 for line in content: 1671 output.write(line) 1672 output.close() 1673 elif not cacheok and os.path.exists(self.mirrorlist_file): 1674 # New mirror file failed, so use the old one (better than nothing) 1675 os.utime(self.mirrorlist_file, None) 1676 return self._readMirrorList(open(self.mirrorlist_file, 'r'))[0] 1677 1678 return returnlist

1679

1680 - def _preload_file(self, fn, destfn):

1681 """attempts to copy the file, if possible""" 1682 # don't copy it if the copy in our users dir is newer or equal 1683 if not os.path.exists(fn): 1684 return False 1685 if os.path.exists(destfn): 1686 if os.stat(fn)[stat.ST_CTIME] <= os.stat(destfn)[stat.ST_CTIME]: 1687 return False 1688 shutil.copy2(fn, destfn) 1689 return True

1690

1691 - def _preload_file_from_system_cache(self, filename, subdir='', 1692 destfn=None):

1693 """attempts to copy the file from the system-wide cache, 1694 if possible""" 1695 if not hasattr(self, 'old_base_cache_dir'): 1696 return False 1697 if self.old_base_cache_dir == "": 1698 return False 1699 1700 glob_repo_cache_dir=os.path.join(self.old_base_cache_dir, self.id) 1701 if not os.path.exists(glob_repo_cache_dir): 1702 return False 1703 if os.path.normpath(glob_repo_cache_dir) == os.path.normpath(self.cachedir): 1704 return False 1705 1706 # Try to copy whatever file it is 1707 fn = glob_repo_cache_dir + '/' + subdir + os.path.basename(filename) 1708 if destfn is None: 1709 destfn = self.cachedir + '/' + subdir + os.path.basename(filename) 1710 return self._preload_file(fn, destfn)

1711

1712 - def _preload_md_from_system_cache(self, filename):

1713 """attempts to copy the metadata file from the system-wide cache, 1714 if possible""" 1715 return self._preload_file_from_system_cache(filename)

1716

1717 - def _preload_pkg_from_system_cache(self, pkg):

1718 """attempts to copy the package from the system-wide cache, 1719 if possible""" 1720 pname = os.path.basename(pkg.localPkg()) 1721 destfn = os.path.join(self.pkgdir, pname) 1722 if self._preload_file_from_system_cache(pkg.localPkg(), 1723 subdir='packages/', 1724 destfn=destfn): 1725 return True 1726 1727 if not hasattr(self, '_old_pkgdirs'): 1728 return False 1729 for opkgdir in self._old_pkgdirs: 1730 if self._preload_file(os.path.join(opkgdir, pname), destfn): 1731 return True 1732 return False

1733

1734 - def _verify_md(self):

1735 problems = [] 1736 print 'verifying md' 1737 try: 1738 md_types = self.repoXML.fileTypes() 1739 except Errors.RepoError, e: 1740 prb = RepoVerifyProblem(1, "failed to load repomd.xml", str(e)) 1741 problems.append(prb) 1742 return problems 1743 1744 for md_type in md_types: 1745 print 'verifying %s' % md_type 1746 try: 1747 self.retrieveMD(md_type) 1748 except Errors.RepoError, e: 1749 msg = "%s metadata missing or does not match checksum" % md_type 1750 prb = RepoVerifyProblem(2, msg, str(e)) 1751 problems.append(prb) 1752 1753 return problems

1754

1755 - def _verify_comps(self):

1756 print 'verifying comps' 1757 problems = [] 1758 # grab the comps for this repo 1759 # run the xmllint on it 1760 # chuck it into a comps object 1761 # make sure it parses 1762 1763 grpfile = self.getGroups() 1764 1765 # open it up as a file object so iterparse can cope with our gz file 1766 if grpfile is not None and grpfile.endswith('.gz'): 1767 grpfile = gzip.open(grpfile) 1768 try: 1769 c = comps.Comps() 1770 c.add(grpfile) 1771 except (Errors.GroupsError, Errors.CompsException), e: 1772 msg = "comps file failed to add" 1773 prb = RepoVerifyProblem(REPO_PROBLEM_COMPS, msg, str(e)) 1774 problems.add(prb) 1775 else: 1776 if c.compscount == 0: 1777 msg = "no groups in comps" 1778 prb = RepoVerifyProblem(REPO_PROBLEM_COMPS, msg, "") 1779 problems.add(prb) 1780 1781 return problems

1782

1783 - def _verify_packages(self):

1784 return []

1785

1786 - def verify(self, items=['repodata', 'comps']):

1787 """download/verify the specified items 1788 @items = ['repodata', 'comps'] can include: repodata, comps, packages 1789 """ 1790 problems = [] 1791 if 'repodata' in items: 1792 problems.extend(self._verify_md()) 1793 if 'comps' in items: 1794 if self.enablegroups: 1795 problems.extend(self._verify_comps()) 1796 if 'packages' in items: 1797 problems.extend(self._verify_packages()) 1798 # what else can we verify? 1799 1800 return problems

1801

1802 1803 -def getMirrorList(mirrorlist, pdict = None):

1804 warnings.warn('getMirrorList() will go away in a future version of Yum.\n', 1805 Errors.YumFutureDeprecationWarning, stacklevel=2) 1806 """retrieve an up2date-style mirrorlist file from a url, 1807 we also s/$ARCH/$BASEARCH/ and move along 1808 returns a list of the urls from that file""" 1809 1810 returnlist = [] 1811 if hasattr(urlgrabber.grabber, 'urlopen'): 1812 urlresolver = urlgrabber.grabber 1813 else: 1814 import urllib 1815 urlresolver = urllib 1816 1817 scheme = urlparse.urlparse(mirrorlist)[0] 1818 if scheme == '': 1819 url = 'file://' + mirrorlist 1820 else: 1821 url = mirrorlist 1822 1823 try: 1824 fo = urlresolver.urlopen(url, proxies=pdict) 1825 except urlgrabber.grabber.URLGrabError, e: 1826 print "Could not retrieve mirrorlist %s error was\n%s: %s" % (url, e.args[0], misc.to_unicode(e.args[1])) 1827 fo = None 1828 1829 if fo is not None: 1830 content = fo.readlines() 1831 for line in content: 1832 if re.match('\s*(#|$)', line): 1833 continue 1834 mirror = line.rstrip() # no more trailing \n's 1835 mirror = mirror.replace('$ARCH', '$BASEARCH') 1836 returnlist.append(mirror) 1837 1838 return returnlist

1839

1840 -class RepoVerifyProblem:

1841 """ Holder for each "problem" we find with a repo.verify(). """ 1842

1843 - def __init__(self, type, msg, details, fake=False):

1844 self.type = type 1845 self.message = msg 1846 self.details = details 1847 self.fake = fake

1848

Source Code for Module yum.yumRepo