yum.metalink

1 #!/usr/bin/python -t 2 # This program is free software; you can redistribute it and/or modify 3 # it under the terms of the GNU General Public License as published by 4 # the Free Software Foundation; either version 2 of the License, or 5 # (at your option) any later version. 6 # 7 # This program is distributed in the hope that it will be useful, 8 # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 # GNU Library General Public License for more details. 11 # 12 # You should have received a copy of the GNU General Public License 13 # along with this program; if not, write to the Free Software 14 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 15 # 16 # Copyright 2008 Red Hat 17 # 18 # James Antill <james@fedoraproject.org> 19 20 # Parse the new MirrorManager metalink output: 21 22 import sys 23 import os 24 import time 25 from urlgrabber.progress import format_number 26 27 import Errors 28 29 try: 30 from xml.etree import cElementTree 31 except ImportError: 32 import cElementTree 33 xmlparse = cElementTree.parse 34

35 -class MetaLinkRepoErrorParseFail(Errors.RepoError):

36 """ An exception thrown for an unparsable MetaLinkRepoMD file. """ 37 pass

38 39 __XML_NS_ML__ = 'http://www.metalinker.org/' 40 __XML_NS_MM__ = 'http://fedorahosted.org/mirrormanager' 41 __XML_FMT__ = {'ml' : __XML_NS_ML__, 42 'mm' : __XML_NS_MM__} 43 44 __ML_FILE_ELEMENT__ = """\ 45 {%(ml)s}files/{%(ml)s}file\ 46 """ % __XML_FMT__ 47 __ML_OLD_FILE_ELEMENTS__ = """\ 48 {%(mm)s}alternates/{%(mm)s}alternate\ 49 """ % __XML_FMT__ 50 __ML_RESOURCES__ = """\ 51 {%(ml)s}resources\ 52 """ % __XML_FMT__ 53

54 -class MetaLinkFile:

55 """ Parse the file metadata out of a metalink file. """ 56

57 - def __init__(self, elem):

58 # We aren't "using" any of these, just storing them. 59 chksums = set(["md5", 'sha1', 'sha256', 'sha512']) 60 61 for celem in elem: 62 if False: pass 63 elif celem.tag == "{%s}timestamp" % __XML_NS_MM__: 64 self.timestamp = int(celem.text) 65 elif celem.tag == "{%s}size" % __XML_NS_ML__: 66 self.size = int(celem.text) 67 elif celem.tag == "{%s}verification" % __XML_NS_ML__: 68 self.chksums = {} 69 for helem in celem: 70 if (helem.tag == "{%s}hash" % __XML_NS_ML__ and 71 helem.get("type") in chksums): 72 self.chksums[helem.get("type").lower()] = helem.text 73 74 if not hasattr(self, 'timestamp'): 75 raise MetaLinkRepoErrorParseFail, "No timestamp for file" 76 if not hasattr(self, 'size'): 77 raise MetaLinkRepoErrorParseFail, "No size for file" 78 if not hasattr(self, 'chksums'): 79 raise MetaLinkRepoErrorParseFail, "No verifications for file"

80

81 - def __str__(self):

82 return """\ 83 Timestamp: %s 84 Size: %5s (%d) 85 MD5: %s 86 SHA1: %s 87 SHA256: %s 88 SHA512: %s 89 """ % (time.ctime(self.timestamp), format_number(self.size), self.size, 90 self.md5, self.sha1, self.sha256, self.sha512)

91

92 - def _get_md5(self):

93 return self.chksums.get('md5', '')

94 md5 = property(_get_md5)

95 - def _get_sha1(self):

96 return self.chksums.get('sha1', '')

97 sha1 = property(_get_sha1)

98 - def _get_sha256(self):

99 return self.chksums.get('sha256', '')

100 sha256 = property(_get_sha256)

101 - def _get_sha512(self):

102 return self.chksums.get('sha512', '')

103 sha512 = property(_get_sha512) 104

105 - def __cmp__(self, other):

106 if other is None: 107 return 1 108 ret = cmp(self.timestamp, other.timestamp) 109 if ret: 110 return -ret 111 ret = cmp(self.size, other.size) 112 if ret: 113 return ret 114 ret = cmp(self.md5, other.md5) 115 if ret: 116 return ret 117 ret = cmp(self.sha1, other.sha1) 118 if ret: 119 return ret 120 ret = cmp(self.sha256, other.sha256) 121 if ret: 122 return ret 123 ret = cmp(self.sha512, other.sha512) 124 if ret: 125 return ret 126 return 0

127 128

129 -class MetaLinkURL:

130 """ Parse the URL metadata out of a metalink file. """ 131

132 - def __init__(self, elem, max_connections):

133 assert elem.tag == '{%s}url' % __XML_NS_ML__ 134 135 self.max_connections = max_connections 136 137 self.url = elem.text 138 self.preference = int(elem.get("preference", -1)) 139 self.protocol = elem.get("type") # This is the "std" attribute name 140 self.location = elem.get("location") 141 142 if self.protocol is None: # Try for the old MM protocol attribute 143 self.protocol = elem.get("protocol")

144

145 - def __str__(self):

146 return """\ 147 URL: %s 148 Preference: %d 149 Max-Connections: %d 150 Protocol: %s 151 Location: %s 152 """ % (self.url, self.preference, self.max_connections, 153 self.protocol, self.location)

154

155 - def __cmp__(self, other):

156 if other is None: 157 return 1 158 ret = cmp(self.preference, other.preference) 159 if ret: 160 return -ret 161 ret = cmp(self.protocol == "https", other.protocol == "https") 162 if ret: 163 return -ret 164 ret = cmp(self.protocol == "http", other.protocol == "http") 165 if ret: 166 return -ret 167 return cmp(self.url, other.url)

168

169 - def usable(self):

170 if self.protocol is None: 171 return False 172 if not self.url: 173 return False 174 return True

175

176 -class MetaLinkRepoMD:

177 """ Parse a metalink file for repomd.xml. """ 178

179 - def __init__(self, filename):

180 self.name = None 181 self.repomd = None 182 self.old_repomds = [] 183 self.mirrors = [] 184 if not os.path.exists(filename): 185 raise MetaLinkRepoErrorParseFail, "File %s does not exist" %filename 186 try: 187 root = xmlparse(filename) 188 except SyntaxError: 189 raise MetaLinkRepoErrorParseFail, "File %s is not XML" % filename 190 191 for elem in root.findall(__ML_FILE_ELEMENT__): 192 name = elem.get('name') 193 if os.path.basename(name) != 'repomd.xml': 194 continue 195 196 if self.name is not None and self.name != name: 197 raise MetaLinkRepoErrorParseFail, "Different paths for repomd file" 198 self.name = name 199 200 repomd = MetaLinkFile(elem) 201 202 if self.repomd is not None and self.repomd != repomd: 203 raise MetaLinkRepoErrorParseFail, "Different data for repomd file" 204 self.repomd = repomd 205 206 for celem in elem.findall(__ML_OLD_FILE_ELEMENTS__): 207 self.old_repomds.append(MetaLinkFile(celem)) 208 209 for celem in elem.findall(__ML_RESOURCES__): 210 max_connections = int(celem.get("maxconnections")) 211 for uelem in celem: 212 if uelem.tag == "{%s}url" % __XML_NS_ML__: 213 self.mirrors.append(MetaLinkURL(uelem, max_connections)) 214 215 self.old_repomds.sort() 216 self.mirrors.sort() 217 218 if self.repomd is None: 219 raise MetaLinkRepoErrorParseFail, "No repomd file" 220 if len(self.mirrors) < 1: 221 raise MetaLinkRepoErrorParseFail, "No mirror"

222

223 - def urls(self):

224 """ Iterate plain urls for the mirrors, like the old mirrorlist. """ 225 226 # Get the hostname from a url, stripping away any usernames/passwords 227 # Borrowd from fastestmirror 228 url2host = lambda url: url.split('/')[2].split('@')[-1] 229 hosts = set() # Don't want multiple urls for one host in plain mode 230 # The list of URLs is sorted, so http is before ftp 231 232 for mirror in self.mirrors: 233 url = mirror.url 234 235 # This is what yum supports atm. ... no rsync etc. 236 if url.startswith("file:"): 237 pass 238 elif (url.startswith("http:") or url.startswith("ftp:") or 239 url.startswith("https:")): 240 host = url2host(url) 241 if host in hosts: 242 continue 243 hosts.add(host) 244 else: 245 continue 246 247 # The mirror urls in the metalink file are for repomd.xml so it 248 # gives a list of mirrors for that one file, but we want the list 249 # of mirror baseurls. Joy of reusing other people's stds. :) 250 if not url.endswith("/repodata/repomd.xml"): 251 continue 252 yield url[:-len("/repodata/repomd.xml")]

253

254 - def __str__(self):

255 ret = str(self.repomd) 256 done = False 257 for orepomd in self.old_repomds: 258 if not done: ret += "%s\n" % ("-" * 79) 259 if done: ret += "\n" 260 done = True 261 ret += str(orepomd) 262 done = False 263 for url in self.mirrors: 264 if not done: ret += "%s\n" % ("-" * 79) 265 if done: ret += "\n" 266 done = True 267 ret += str(url) 268 return ret

269 270

271 -def main():

272 """ MetaLinkRepoMD test function. """ 273 274 def usage(): 275 print >> sys.stderr, "Usage: %s <metalink> ..." % sys.argv[0] 276 sys.exit(1)

277 278 if len(sys.argv) < 2: 279 usage() 280 281 for filename in sys.argv[1:]: 282 if not os.path.exists(filename): 283 print "No such file:", filename 284 continue 285 286 print "File:", filename 287 print MetaLinkRepoMD(filename) 288 print '' 289 290 if __name__ == '__main__': 291 main() 292

Source Code for Module yum.metalink